Commit my checkpoint of db_disasm.c that I hacked to understand some amd64

instructions as it was when I dropped it back in May 31, 2003.  I'm
committing this as an intermediate stage because back then I thought I
understood what I was doing with this file.

1 files changed, 116 insertions, 79 deletions

diff --git a/sys/amd64/amd64/db_disasm.c b/sys/amd64/amd64/db_disasm.c
index 66a3c4b..9717ae1 100644
--- a/sys/amd64/amd64/db_disasm.c
+++ b/sys/amd64/amd64/db_disasm.c
@@ -50,6 +50,15 @@ __FBSDID("$FreeBSD$");
 #define	NONE	8
 
 /*
+ * REX prefix and bits
+ */
+#define REX_B	1
+#define REX_X	2
+#define REX_R	4
+#define REX_W	8
+#define REX	0x40
+
+/*
  * Addressing modes
  */
 #define	E	1			/* general effective address */
@@ -585,23 +594,23 @@ static const struct inst db_inst_table[256] = {
 /*3e*/	{ "",      FALSE, NONE,  0,	     0 },
 /*3f*/	{ "aas",   FALSE, NONE,  0,	     0 },
 
-/*40*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*41*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*42*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*43*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*44*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*45*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*46*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-/*47*/	{ "inc",   FALSE, LONG,  op1(Ri),    0 },
-
-/*48*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*49*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*4a*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*4b*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*4c*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*4d*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*4e*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
-/*4f*/	{ "dec",   FALSE, LONG,  op1(Ri),    0 },
+/*40*/	{ "rex",   FALSE, NONE,  0,          0 },
+/*41*/	{ "rex.b", FALSE, NONE,  0,          0 },
+/*42*/	{ "rex.x", FALSE, NONE,  0,          0 },
+/*43*/	{ "rex.xb", FALSE, NONE, 0,          0 },
+/*44*/	{ "rex.r", FALSE, NONE,  0,          0 },
+/*45*/	{ "rex.rb", FALSE, NONE, 0,          0 },
+/*46*/	{ "rex.rx", FALSE, NONE, 0,          0 },
+/*47*/	{ "rex.rxb", FALSE, NONE, 0,         0 },
+
+/*48*/	{ "rex.w", FALSE, NONE,  0,          0 },
+/*49*/	{ "rex.wb", FALSE, NONE, 0,          0 },
+/*4a*/	{ "rex.wx", FALSE, NONE, 0,          0 },
+/*4b*/	{ "rex.wxb", FALSE, NONE, 0,         0 },
+/*4c*/	{ "rex.wr", FALSE, NONE, 0,          0 },
+/*4d*/	{ "rex.wrb", FALSE, NONE, 0,         0 },
+/*4e*/	{ "rex.wrx", FALSE, NONE, 0,         0 },
+/*4f*/	{ "rex.wrxb", FALSE, NONE, 0,        0 },
 
 /*50*/	{ "push",  FALSE, LONG,  op1(Ri),    0 },
 /*51*/	{ "push",  FALSE, LONG,  op1(Ri),    0 },
@@ -807,13 +816,13 @@ static const struct inst db_bad_inst =
 	{ "???",   FALSE, NONE,  0,	      0 }
 ;
 
-#define	f_mod(byte)	((byte)>>6)
-#define	f_reg(byte)	(((byte)>>3)&0x7)
-#define	f_rm(byte)	((byte)&0x7)
+#define	f_mod(rex, byte)	((byte)>>6)
+#define	f_reg(rex, byte)	((((byte)>>3)&0x7) | (rex & REX_R ? 0x8 : 0x0))
+#define	f_rm(rex, byte)		(((byte)&0x7) | (rex & REX_B ? 0x8 : 0x0))
 
-#define	sib_ss(byte)	((byte)>>6)
-#define	sib_index(byte)	(((byte)>>3)&0x7)
-#define	sib_base(byte)	((byte)&0x7)
+#define	sib_ss(rex, byte)	((byte)>>6)
+#define	sib_index(rex, byte)	((((byte)>>3)&0x7) | (rex & REX_X ? 0x8 : 0x0))
+#define	sib_base(rex, byte)	(((byte)&0x7) | (rex & REX_B ? 0x8 : 0x0))
 
 struct i_addr {
 	int		is_reg;	/* if reg, reg number is in 'disp' */
@@ -834,10 +843,25 @@ static const char * const db_index_reg_16[8] = {
 	"%bx"
 };
 
-static const char * const db_reg[3][8] = {
-	{ "%al",  "%cl",  "%dl",  "%bl",  "%ah",  "%ch",  "%dh",  "%bh" },
-	{ "%ax",  "%cx",  "%dx",  "%bx",  "%sp",  "%bp",  "%si",  "%di" },
-	{ "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi" }
+static const char * const db_reg[2][4][16] = {
+
+	{{"%al",  "%cl",  "%dl",  "%bl",  "%ah",  "%ch",  "%dh",  "%bh",
+	  "%r8b", "%r9b", "%r10b", "%r11b", "%r12b", "%r13b", "%r14b", "%r15b" },
+	{ "%ax",  "%cx",  "%dx",  "%bx",  "%sp",  "%bp",  "%si",  "%di",
+	  "%r8w", "%r9w", "%r10w", "%r11w", "%r12w", "%r13w", "%r14w", "%r15w" },
+	{ "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
+	  "%r8d", "%r9d", "%r10d", "%r11d", "%r12d", "%r13d", "%r14d", "%r15d" },
+	{ "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
+	  "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15" }},
+
+	{{"%al",  "%cl",  "%dl",  "%bl",  "%spl",  "%bpl",  "%sil",  "%dil",
+	  "%r8b", "%r9b", "%r10b", "%r11b", "%r12b", "%r13b", "%r14b", "%r15b" },
+	{ "%ax",  "%cx",  "%dx",  "%bx",  "%sp",  "%bp",  "%si",  "%di",
+	  "%r8w", "%r9w", "%r10w", "%r11w", "%r12w", "%r13w", "%r14w", "%r15w" },
+	{ "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
+	  "%r8d", "%r9d", "%r10d", "%r11d", "%r12d", "%r13d", "%r14d", "%r15d" },
+	{ "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
+	  "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15" }}
 };
 
 static const char * const db_seg_reg[8] = {
@@ -857,33 +881,35 @@ static const int db_lengths[] = {
 	10,	/* EXTR */
 };
 
+
 #define	get_value_inc(result, loc, size, is_signed) \
 	result = db_get_value((loc), (size), (is_signed)); \
 	(loc) += (size);
 
 static db_addr_t
-		db_disasm_esc(db_addr_t loc, int inst, int short_addr,
+		db_disasm_esc(db_addr_t loc, int inst, int rex, int short_addr,
 		    int size, const char *seg);
-static void	db_print_address(const char *seg, int size,
+static void	db_print_address(const char *seg, int size, int rex,
 		    struct i_addr *addrp);
 static db_addr_t
-		db_read_address(db_addr_t loc, int short_addr, int regmodrm,
+		db_read_address(db_addr_t loc, int short_addr, int rex, int regmodrm,
 		    struct i_addr *addrp);
 
 /*
  * Read address at location and return updated location.
  */
 static db_addr_t
-db_read_address(loc, short_addr, regmodrm, addrp)
+db_read_address(loc, short_addr, rex, regmodrm, addrp)
 	db_addr_t	loc;
 	int		short_addr;
+	int		rex;
 	int		regmodrm;
 	struct i_addr *	addrp;		/* out */
 {
 	int		mod, rm, sib, index, disp;
 
-	mod = f_mod(regmodrm);
-	rm  = f_rm(regmodrm);
+	mod = f_mod(rex, regmodrm);
+	rm  = f_rm(rex, regmodrm);
 
 	if (mod == 3) {
 	    addrp->is_reg = TRUE;
@@ -924,11 +950,11 @@ db_read_address(loc, short_addr, regmodrm, addrp)
 	else {
 	    if (mod != 3 && rm == 4) {
 		get_value_inc(sib, loc, 1, FALSE);
-		rm = sib_base(sib);
-		index = sib_index(sib);
+		rm = sib_base(rex, sib);
+		index = sib_index(rex, sib);
 		if (index != 4)
-		    addrp->index = db_reg[LONG][index];
-		addrp->ss = sib_ss(sib);
+		    addrp->index = db_reg[rex != 0 ? 1 : 0][(rex & REX_R) ? QUAD : LONG][index];
+		addrp->ss = sib_ss(rex, sib);
 	    }
 
 	    switch (mod) {
@@ -939,20 +965,20 @@ db_read_address(loc, short_addr, regmodrm, addrp)
 		    }
 		    else {
 			addrp->disp = 0;
-			addrp->base = db_reg[LONG][rm];
+			addrp->base = db_reg[rex != 0 ? 1 : 0][(rex & REX_R) ? QUAD : LONG][rm];
 		    }
 		    break;
 
 		case 1:
 		    get_value_inc(disp, loc, 1, TRUE);
 		    addrp->disp = disp;
-		    addrp->base = db_reg[LONG][rm];
+		    addrp->base = db_reg[rex != 0 ? 1 : 0][(rex & REX_R) ? QUAD : LONG][rm];
 		    break;
 
 		case 2:
 		    get_value_inc(disp, loc, 4, FALSE);
 		    addrp->disp = disp;
-		    addrp->base = db_reg[LONG][rm];
+		    addrp->base = db_reg[rex != 0 ? 1 : 0][(rex & REX_R) ? QUAD : LONG][rm];
 		    break;
 	    }
 	}
@@ -960,13 +986,14 @@ db_read_address(loc, short_addr, regmodrm, addrp)
 }
 
 static void
-db_print_address(seg, size, addrp)
+db_print_address(seg, size, rex, addrp)
 	const char *	seg;
 	int		size;
+	int		rex;
 	struct i_addr *	addrp;
 {
 	if (addrp->is_reg) {
-	    db_printf("%s", db_reg[size][addrp->disp]);
+	    db_printf("%s", db_reg[rex != 0 ? 1 : 0][size][addrp->disp]);
 	    return;
 	}
 
@@ -990,9 +1017,10 @@ db_print_address(seg, size, addrp)
  * and return updated location.
  */
 static db_addr_t
-db_disasm_esc(loc, inst, short_addr, size, seg)
+db_disasm_esc(loc, inst, rex, short_addr, size, seg)
 	db_addr_t	loc;
 	int		inst;
+	int		rex;
 	int		short_addr;
 	int		size;
 	const char *	seg;
@@ -1004,8 +1032,8 @@ db_disasm_esc(loc, inst, short_addr, size, seg)
 	const char *	name;
 
 	get_value_inc(regmodrm, loc, 1, FALSE);
-	fp = &db_Esc_inst[inst - 0xd8][f_reg(regmodrm)];
-	mod = f_mod(regmodrm);
+	fp = &db_Esc_inst[inst - 0xd8][f_reg(rex, regmodrm)];
+	mod = f_mod(rex, regmodrm);
 	if (mod != 3) {
 	    if (*fp->f_name == '\0') {
 		db_printf("<bad instruction>");
@@ -1014,7 +1042,7 @@ db_disasm_esc(loc, inst, short_addr, size, seg)
 	    /*
 	     * Normal address modes.
 	     */
-	    loc = db_read_address(loc, short_addr, regmodrm, &address);
+	    loc = db_read_address(loc, short_addr, rex, regmodrm, &address);
 	    db_printf("%s", fp->f_name);
 	    switch(fp->f_size) {
 		case SNGL:
@@ -1039,7 +1067,7 @@ db_disasm_esc(loc, inst, short_addr, size, seg)
 		    break;
 	    }
 	    db_printf("\t");
-	    db_print_address(seg, BYTE, &address);
+	    db_print_address(seg, BYTE, rex, &address);
 	}
 	else {
 	    /*
@@ -1048,24 +1076,24 @@ db_disasm_esc(loc, inst, short_addr, size, seg)
 	    switch (fp->f_rrmode) {
 		case op2(ST,STI):
 		    name = (fp->f_rrname) ? fp->f_rrname : fp->f_name;
-		    db_printf("%s\t%%st,%%st(%d)",name,f_rm(regmodrm));
+		    db_printf("%s\t%%st,%%st(%d)",name,f_rm(rex, regmodrm));
 		    break;
 		case op2(STI,ST):
 		    name = (fp->f_rrname) ? fp->f_rrname : fp->f_name;
-		    db_printf("%s\t%%st(%d),%%st",name, f_rm(regmodrm));
+		    db_printf("%s\t%%st(%d),%%st",name, f_rm(rex, regmodrm));
 		    break;
 		case op1(STI):
 		    name = (fp->f_rrname) ? fp->f_rrname : fp->f_name;
-		    db_printf("%s\t%%st(%d)",name, f_rm(regmodrm));
+		    db_printf("%s\t%%st(%d)",name, f_rm(rex, regmodrm));
 		    break;
 		case op1(X):
-		    name = ((const char * const *)fp->f_rrname)[f_rm(regmodrm)];
+		    name = ((const char * const *)fp->f_rrname)[f_rm(rex, regmodrm)];
 		    if (*name == '\0')
 			goto bad;
 		    db_printf("%s", name);
 		    break;
 		case op1(XA):
-		    name = ((const char * const *)fp->f_rrname)[f_rm(regmodrm)];
+		    name = ((const char * const *)fp->f_rrname)[f_rm(rex, regmodrm)];
 		    if (*name == '\0')
 			goto bad;
 		    db_printf("%s\t%%ax", name);
@@ -1098,6 +1126,7 @@ db_disasm(loc, altfmt)
 	const char *	i_name;
 	int	i_size;
 	int	i_mode;
+	int	rex = 0;
 	int	regmodrm = 0;
 	boolean_t	first;
 	int	displ;
@@ -1155,13 +1184,17 @@ db_disasm(loc, altfmt)
 		    prefix = FALSE;
 		    break;
 	    }
+	    if (inst >= 0x40 && inst < 0x50) {
+		rex = inst;
+		prefix = TRUE;
+	    }
 	    if (prefix) {
 		get_value_inc(inst, loc, 1, FALSE);
 	    }
 	} while (prefix);
 
 	if (inst >= 0xd8 && inst <= 0xdf) {
-	    loc = db_disasm_esc(loc, inst, short_addr, size, seg);
+	    loc = db_disasm_esc(loc, inst, rex, short_addr, size, seg);
 	    db_printf("\n");
 	    return (loc);
 	}
@@ -1181,7 +1214,7 @@ db_disasm(loc, altfmt)
 
 	if (ip->i_has_modrm) {
 	    get_value_inc(regmodrm, loc, 1, FALSE);
-	    loc = db_read_address(loc, short_addr, regmodrm, &address);
+	    loc = db_read_address(loc, short_addr, rex, regmodrm, &address);
 	}
 
 	i_name = ip->i_name;
@@ -1191,17 +1224,17 @@ db_disasm(loc, altfmt)
 	if (ip->i_extra == db_Grp1 || ip->i_extra == db_Grp2 ||
 	    ip->i_extra == db_Grp6 || ip->i_extra == db_Grp7 ||
 	    ip->i_extra == db_Grp8 || ip->i_extra == db_Grp9) {
-	    i_name = ((const char * const *)ip->i_extra)[f_reg(regmodrm)];
+	    i_name = ((const char * const *)ip->i_extra)[f_reg(rex, regmodrm)];
 	}
 	else if (ip->i_extra == db_Grp3) {
 	    ip = ip->i_extra;
-	    ip = &ip[f_reg(regmodrm)];
+	    ip = &ip[f_reg(rex, regmodrm)];
 	    i_name = ip->i_name;
 	    i_mode = ip->i_mode;
 	}
 	else if (ip->i_extra == db_Grp4 || ip->i_extra == db_Grp5) {
 	    ip = ip->i_extra;
-	    ip = &ip[f_reg(regmodrm)];
+	    ip = &ip[f_reg(rex, regmodrm)];
 	    i_name = ip->i_name;
 	    i_mode = ip->i_mode;
 	    i_size = ip->i_size;
@@ -1226,8 +1259,12 @@ db_disasm(loc, altfmt)
 		}
 		else if (size == WORD)
 		    db_printf("w");
-		else
-		    db_printf("l");
+		else {
+		    if (rex & REX_W)
+			db_printf("q");
+		    else
+			db_printf("l");
+		}
 	    }
 	}
 	db_printf("\t");
@@ -1241,52 +1278,52 @@ db_disasm(loc, altfmt)
 	    switch (i_mode & 0xFF) {
 
 		case E:
-		    db_print_address(seg, size, &address);
+		    db_print_address(seg, size, rex, &address);
 		    break;
 
 		case Eind:
 		    db_printf("*");
-		    db_print_address(seg, size, &address);
+		    db_print_address(seg, size, rex, &address);
 		    break;
 
 		case El:
-		    db_print_address(seg, LONG, &address);
+		    db_print_address(seg, (rex & REX_W) ? QUAD : LONG, rex, &address);
 		    break;
 
 		case Ew:
-		    db_print_address(seg, WORD, &address);
+		    db_print_address(seg, WORD, rex, &address);
 		    break;
 
 		case Eb:
-		    db_print_address(seg, BYTE, &address);
+		    db_print_address(seg, BYTE, rex, &address);
 		    break;
 
 		case R:
-		    db_printf("%s", db_reg[size][f_reg(regmodrm)]);
+		    db_printf("%s", db_reg[rex != 0 ? 1 : 0][size][f_reg(rex, regmodrm)]);
 		    break;
 
 		case Rw:
-		    db_printf("%s", db_reg[WORD][f_reg(regmodrm)]);
+		    db_printf("%s", db_reg[rex != 0 ? 1 : 0][WORD][f_reg(rex, regmodrm)]);
 		    break;
 
 		case Ri:
-		    db_printf("%s", db_reg[size][f_rm(inst)]);
+		    db_printf("%s", db_reg[rex != 0 ? 1 : 0][size][f_rm(rex, inst)]);
 		    break;
 
 		case Ril:
-		    db_printf("%s", db_reg[LONG][f_rm(inst)]);
+		    db_printf("%s", db_reg[rex != 0 ? 1 : 0][(rex & REX_R) ? QUAD : LONG][f_rm(rex, inst)]);
 		    break;
 
 		case S:
-		    db_printf("%s", db_seg_reg[f_reg(regmodrm)]);
+		    db_printf("%s", db_seg_reg[f_reg(rex, regmodrm)]);
 		    break;
 
 		case Si:
-		    db_printf("%s", db_seg_reg[f_reg(inst)]);
+		    db_printf("%s", db_seg_reg[f_reg(rex, inst)]);
 		    break;
 
 		case A:
-		    db_printf("%s", db_reg[size][0]);	/* acc */
+		    db_printf("%s", db_reg[rex != 0 ? 1 : 0][size][0]);	/* acc */
 		    break;
 
 		case BX:
@@ -1306,33 +1343,33 @@ db_disasm(loc, altfmt)
 		case SI:
 		    if (seg)
 			db_printf("%s:", seg);
-		    db_printf("(%s)", short_addr ? "%si" : "%esi");
+		    db_printf("(%s)", short_addr ? "%si" : "%rsi");
 		    break;
 
 		case DI:
-		    db_printf("%%es:(%s)", short_addr ? "%di" : "%edi");
+		    db_printf("%%es:(%s)", short_addr ? "%di" : "%rdi");
 		    break;
 
 		case CR:
-		    db_printf("%%cr%d", f_reg(regmodrm));
+		    db_printf("%%cr%d", f_reg(rex, regmodrm));
 		    break;
 
 		case DR:
-		    db_printf("%%dr%d", f_reg(regmodrm));
+		    db_printf("%%dr%d", f_reg(rex, regmodrm));
 		    break;
 
 		case TR:
-		    db_printf("%%tr%d", f_reg(regmodrm));
+		    db_printf("%%tr%d", f_reg(rex, regmodrm));
 		    break;
 
 		case I:
-		    len = db_lengths[size];
+		    len = db_lengths[(size == LONG && (rex & REX_W)) ? QUAD : size];
 		    get_value_inc(imm, loc, len, FALSE);
 		    db_printf("$%#r", imm);
 		    break;
 
 		case Is:
-		    len = db_lengths[size];
+		    len = db_lengths[(size == LONG && (rex & REX_W)) ? QUAD : size];
 		    get_value_inc(imm, loc, len, FALSE);
 		    db_printf("$%+#r", imm);
 		    break;
@@ -1378,7 +1415,7 @@ db_disasm(loc, altfmt)
 		    break;
 
 		case Dl:
-		    len = db_lengths[size];
+		    len = db_lengths[(size == LONG && (rex & REX_W)) ? QUAD : size];
 		    get_value_inc(displ, loc, len, FALSE);
 		    displ += loc;
 		    if (size == WORD)