Changeset 309 for trunk/openjdk/hotspot/src/cpu/x86

trunk/openjdk

Property svn:mergeinfo changed
/branches/vendor/oracle/openjdk6/b24 (added) merged: 308
/branches/vendor/oracle/openjdk6/current merged: 307

trunk/openjdk/hotspot/src/cpu/x86/vm/assembler_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_assembler_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "gc_interface/collectedHeap.inline.hpp"
+#include "interpreter/interpreter.hpp"
+#include "memory/cardTableModRefBS.hpp"
+#include "memory/resourceArea.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/biasedLocking.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "runtime/objectMonitor.hpp"
+#include "runtime/os.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#ifndef SERIALGC
+#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
+#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
+#include "gc_implementation/g1/heapRegion.hpp"
+#endif
 // Implementation of AddressLiteral
 …
+// Now the Assembler instruction (identical for 32/64 bits)
+// Now the Assembler instructions (identical for 32/64 bits)
+void Assembler::adcl(Address dst, int32_t imm32) {
+  InstructionMark im(this);
+  prefix(dst);
+  emit_arith_operand(0x81, rdx, dst, imm32);
+}
+void Assembler::adcl(Address dst, Register src) {
+  InstructionMark im(this);
+  prefix(dst, src);
+  emit_byte(0x11);
+  emit_operand(src, dst);
+}
 void Assembler::adcl(Register dst, int32_t imm32) {
 …
+}
+void Assembler::divl(Register src) { // Unsigned
+  int encode = prefix_and_encode(src->encoding());
+  emit_byte(0xF7);
+  emit_byte(0xF0 | encode);
+}
 void Assembler::imull(Register dst, Register src) {
   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 …
     emit_byte(0x6B);
     emit_byte(0xC0 | encode);
     emit_byte(value);
+    emit_byte(value & 0xFF);
   } else {
     emit_byte(0x69);
 …
   InstructionMark im(this);
   prefix(dst);
+  emit_byte(0x81);
+  emit_operand(rcx, dst, 4);
+  emit_long(imm32);
+  emit_arith_operand(0x81, rcx, dst, imm32);
+}
 …
+}
 void Assembler::orl(Register dst, Address src) {
   InstructionMark im(this);
 …
   emit_operand(dst, src);
+}
 void Assembler::orl(Register dst, Register src) {
 …
 void Assembler::prefetchr(Address src) {
   NOT_LP64(assert(VM_Version::supports_3dnow(), "must support"));
+  NOT_LP64(assert(VM_Version::supports_3dnow_prefetch(), "must support"));
   InstructionMark im(this);
   prefetch_prefix(src);
 …
 void Assembler::prefetchw(Address src) {
   NOT_LP64(assert(VM_Version::supports_3dnow(), "must support"));
+  NOT_LP64(assert(VM_Version::supports_3dnow_prefetch(), "must support"));
   InstructionMark im(this);
   prefetch_prefix(src);
 …
 void Assembler::prefix(Prefix p) {
   a_byte(p);
+}
+void Assembler::por(XMMRegister dst, XMMRegister src) {
+  NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+  emit_byte(0x66);
+  int  encode = prefix_and_encode(dst->encoding(), src->encoding());
+  emit_byte(0x0F);
+  emit_byte(0xEB);
+  emit_byte(0xC0 | encode);
+}
 …
+}
+void Assembler::sqrtsd(XMMRegister dst, Address src) {
+  NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+  InstructionMark im(this);
+  emit_byte(0xF2);
+  prefix(src, dst);
+  emit_byte(0x0F);
+  emit_byte(0x51);
+  emit_operand(dst, src);
+}
+void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
+  // HMM Table D-1 says sse2
+  // NOT_LP64(assert(VM_Version::supports_sse(), ""));
+  NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+  emit_byte(0xF3);
+  int encode = prefix_and_encode(dst->encoding(), src->encoding());
+  emit_byte(0x0F);
+  emit_byte(0x51);
+  emit_byte(0xC0 | encode);
+}
+void Assembler::sqrtss(XMMRegister dst, Address src) {
+  NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+  InstructionMark im(this);
+  emit_byte(0xF3);
+  prefix(src, dst);
+  emit_byte(0x0F);
+  emit_byte(0x51);
+  emit_operand(dst, src);
+}
 void Assembler::stmxcsr( Address dst) {
   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 …
   InstructionMark im(this);
   prefix(dst);
+  if (is8bit(imm32)) {
+    emit_byte(0x83);
+    emit_operand(rbp, dst, 1);
+    emit_byte(imm32 & 0xFF);
+  } else {
+    emit_byte(0x81);
+    emit_operand(rbp, dst, 4);
+    emit_long(imm32);
+  }
+  emit_arith_operand(0x81, rbp, dst, imm32);
+}
+void Assembler::subl(Address dst, Register src) {
+  InstructionMark im(this);
+  prefix(dst, src);
+  emit_byte(0x29);
+  emit_operand(src, dst);
+}
 …
   prefix(dst);
   emit_arith(0x81, 0xE8, dst, imm32);
+}
-void Assembler::subl(Address dst, Register src) {
-  InstructionMark im(this);
-  prefix(dst, src);
-  emit_byte(0x29);
-  emit_operand(src, dst);
+}
 …
     emit_byte(0x6B);
     emit_byte(0xC0 | encode);
     emit_byte(value);
+    emit_byte(value & 0xFF);
   } else {
     emit_byte(0x69);
 …
   emit_byte(0xF8 | encode);
+}
 void Assembler::sbbq(Address dst, int32_t imm32) {
   InstructionMark im(this);
 …
+}
-void Assembler::sqrtsd(XMMRegister dst, Address src) {
-  NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-  InstructionMark im(this);
-  emit_byte(0xF2);
-  prefix(src, dst);
-  emit_byte(0x0F);
-  emit_byte(0x51);
-  emit_operand(dst, src);
+}
 void Assembler::subq(Address dst, int32_t imm32) {
   InstructionMark im(this);
   prefixq(dst);
+  if (is8bit(imm32)) {
+    emit_byte(0x83);
+    emit_operand(rbp, dst, 1);
+    emit_byte(imm32 & 0xFF);
+  } else {
+    emit_byte(0x81);
+    emit_operand(rbp, dst, 4);
+    emit_long(imm32);
+  }
+  emit_arith_operand(0x81, rbp, dst, imm32);
+}
+void Assembler::subq(Address dst, Register src) {
+  InstructionMark im(this);
+  prefixq(dst, src);
+  emit_byte(0x29);
+  emit_operand(src, dst);
+}
 …
   (void) prefixq_and_encode(dst->encoding());
   emit_arith(0x81, 0xE8, dst, imm32);
+}
-void Assembler::subq(Address dst, Register src) {
-  InstructionMark im(this);
-  prefixq(dst, src);
-  emit_byte(0x29);
-  emit_operand(src, dst);
+}
 …
-void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
-  movsd(dst, as_Address(src));
+}
 void MacroAssembler::pop_callee_saved_registers() {
   pop(rcx);
 …
       tty->print_cr("eip = 0x%08x", eip);
 #ifndef PRODUCT
+      tty->cr();
+      findpc(eip);
+      tty->cr();
+      if ((WizardMode || Verbose) && PrintMiscellaneous) {
+        tty->cr();
+        findpc(eip);
+        tty->cr();
+      }
 #endif
       tty->print_cr("rax, = 0x%08x", rax);
       tty->print_cr("rbx, = 0x%08x", rbx);
+      tty->print_cr("rax = 0x%08x", rax);
+      tty->print_cr("rbx = 0x%08x", rbx);
       tty->print_cr("rcx = 0x%08x", rcx);
       tty->print_cr("rdx = 0x%08x", rdx);
       tty->print_cr("rdi = 0x%08x", rdi);
       tty->print_cr("rsi = 0x%08x", rsi);
       tty->print_cr("rbp, = 0x%08x", rbp);
+      tty->print_cr("rbp = 0x%08x", rbp);
       tty->print_cr("rsp = 0x%08x", rsp);
       BREAKPOINT;
+      assert(false, "start up GDB");
+    }
   } else {
 …
 void MacroAssembler::warn(const char* msg) {
+  push(r12);
+  movq(r12, rsp);
+  push(rsp);
   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 …
   call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
   pop_CPU_state();
+  movq(rsp, r12);
+  pop(r12);
+  pop(rsp);
+}
 …
   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
+#ifdef ASSERT
+  LP64_ONLY(if (UseCompressedOops) verify_heapbase("call_VM_base");)
+#endif // ASSERT
   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 …
 // Preserves rbx, and rdx.
 void MacroAssembler::tlab_refill(Label& retry,
                                  Label& try_eden,
                                  Label& slow_case) {
+Register MacroAssembler::tlab_refill(Label& retry,
+                                     Label& try_eden,
+                                     Label& slow_case) {
   Register top = rax;
   Register t1  = rcx;
 …
   // if tlab is currently allocated (top or end != null) then
   // fill [top, end + alignment_reserve) with array object
   testptr (top, top);
+  testptr(top, top);
   jcc(Assembler::zero, do_refill);
 …
   // set klass to intArrayKlass
   // dubious reloc why not an oop reloc?
   movptr(t1, ExternalAddress((address) Universe::intArrayKlassObj_addr()));
+  movptr(t1, ExternalAddress((address)Universe::intArrayKlassObj_addr()));
   // store klass last.  concurrent gcs assumes klass length is valid if
   // klass field is not null.
   store_klass(top, t1);
+  movptr(t1, top);
+  subptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
+  incr_allocated_bytes(thread_reg, t1, 0);
   // refill the tlab with an eden allocation
 …
   movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
   shlptr(t1, LogHeapWordSize);
   // add object_size ??
+  // allocate new tlab, address returned in top
   eden_allocate(top, t1, 0, t2, slow_case);
 …
   verify_tlab();
   jmp(retry);
+  return thread_reg; // for use by caller
+}
+void MacroAssembler::incr_allocated_bytes(Register thread,
+                                          Register var_size_in_bytes,
+                                          int con_size_in_bytes,
+                                          Register t1) {
+#ifdef _LP64
+  if (var_size_in_bytes->is_valid()) {
+    addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
+  } else {
+    addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
+  }
+#else
+  if (!thread->is_valid()) {
+    assert(t1->is_valid(), "need temp reg");
+    thread = t1;
+    get_thread(thread);
+  }
+  if (var_size_in_bytes->is_valid()) {
+    addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
+  } else {
+    addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
+  }
+  adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
+#endif
+}
 …
 #ifdef ASSERT
+  Label L;
+  testptr(tmp, tmp);
+  jccb(Assembler::notZero, L);
+  hlt();
+  bind(L);
+  { Label L;
+    testptr(tmp, tmp);
+    if (WizardMode) {
+      jcc(Assembler::notZero, L);
+      char* buf = new char[40];
+      sprintf(buf, "DelayedValue="INTPTR_FORMAT, delayed_value_addr[1]);
+      stop(buf);
+    } else {
+      jccb(Assembler::notZero, L);
+      hlt();
+    }
+    bind(L);
+  }
 #endif
 …
                                               Register temp_reg,
                                               Label& wrong_method_type) {
   if (UseCompressedOops)  unimplemented();  // field accesses must decode
+  Address type_addr(mh_reg, delayed_value(java_dyn_MethodHandle::type_offset_in_bytes, temp_reg));
   // compare method type against that of the receiver
+  cmpptr(mtype_reg, Address(mh_reg, delayed_value(java_dyn_MethodHandle::type_offset_in_bytes, temp_reg)));
+  if (UseCompressedOops) {
+    load_heap_oop(temp_reg, type_addr);
+    cmpptr(mtype_reg, temp_reg);
+  } else {
+    cmpptr(mtype_reg, type_addr);
+  }
   jcc(Assembler::notEqual, wrong_method_type);
+}
 …
                                                 Register temp_reg) {
   assert_different_registers(vmslots_reg, mh_reg, temp_reg);
-  if (UseCompressedOops)  unimplemented();  // field accesses must decode
   // load mh.type.form.vmslots
   if (java_dyn_MethodHandle::vmslots_offset_in_bytes() != 0) {
 …
   } else {
     Register temp2_reg = vmslots_reg;
     movptr(temp2_reg, Address(mh_reg,    delayed_value(java_dyn_MethodHandle::type_offset_in_bytes, temp_reg)));
     movptr(temp2_reg, Address(temp2_reg, delayed_value(java_dyn_MethodType::form_offset_in_bytes, temp_reg)));
+    load_heap_oop(temp2_reg, Address(mh_reg,    delayed_value(java_dyn_MethodHandle::type_offset_in_bytes, temp_reg)));
+    load_heap_oop(temp2_reg, Address(temp2_reg, delayed_value(java_dyn_MethodType::form_offset_in_bytes, temp_reg)));
     movl(vmslots_reg, Address(temp2_reg, delayed_value(java_dyn_MethodTypeForm::vmslots_offset_in_bytes, temp_reg)));
+  }
 …
   assert_different_registers(mh_reg, temp_reg);
-  if (UseCompressedOops)  unimplemented();  // field accesses must decode
   // pick out the interpreted side of the handler
+  // NOTE: vmentry is not an oop!
   movptr(temp_reg, Address(mh_reg, delayed_value(java_dyn_MethodHandle::vmentry_offset_in_bytes, temp_reg)));
 …
+}
+void MacroAssembler::load_heap_oop(Register dst, Address src) {
+#ifdef _LP64
+  if (UseCompressedOops) {
+    movl(dst, src);
+    decode_heap_oop(dst);
+  } else
+#endif
+    movptr(dst, src);
+}
+void MacroAssembler::store_heap_oop(Address dst, Register src) {
+#ifdef _LP64
+  if (UseCompressedOops) {
+    assert(!dst.uses(src), "not enough registers");
+    encode_heap_oop(src);
+    movl(dst, src);
+  } else
+#endif
+    movptr(dst, src);
+}
+// Used for storing NULLs.
+void MacroAssembler::store_heap_oop_null(Address dst) {
+#ifdef _LP64
+  if (UseCompressedOops) {
+    movl(dst, (int32_t)NULL_WORD);
+  } else {
+    movslq(dst, (int32_t)NULL_WORD);
+  }
+#else
+  movl(dst, (int32_t)NULL_WORD);
+#endif
+}
 #ifdef _LP64
 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 …
     // Store to klass gap in destination
     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
+  }
+}
-void MacroAssembler::load_heap_oop(Register dst, Address src) {
-  if (UseCompressedOops) {
-    movl(dst, src);
-    decode_heap_oop(dst);
-  } else {
-    movq(dst, src);
+  }
+}
-void MacroAssembler::store_heap_oop(Address dst, Register src) {
-  if (UseCompressedOops) {
-    assert(!dst.uses(src), "not enough registers");
-    encode_heap_oop(src);
-    movl(dst, src);
-  } else {
-    movq(dst, src);
+  }
+}
-// Used for storing NULLs.
-void MacroAssembler::store_heap_oop_null(Address dst) {
-  if (UseCompressedOops) {
-    movl(dst, (int32_t)NULL_WORD);
-  } else {
-    movslq(dst, (int32_t)NULL_WORD);
+  }
+}
 …
 void MacroAssembler::string_compare(Register str1, Register str2,
                                     Register cnt1, Register cnt2, Register result,
                                     XMMRegister vec1, XMMRegister vec2) {
+                                    XMMRegister vec1) {
   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
 …
+  }
+  // Advance to next character
+  addptr(str1, 2);
+  addptr(str2, 2);
+  Address::ScaleFactor scale = Address::times_2;
+  int stride = 8;
+  // Advance to next element
+  addptr(str1, 16/stride);
+  addptr(str2, 16/stride);
   if (UseSSE42Intrinsics) {
     // With SSE4.2, use double quad vector compare
     Label COMPARE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
+    Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
+    int pcmpmask = 0x19;
     // Setup to compare 16-byte vectors
+    movl(cnt1, cnt2);
+    andl(cnt2, 0xfffffff8); // cnt2 holds the vector count
+    andl(cnt1, 0x00000007); // cnt1 holds the tail count
+    testl(cnt2, cnt2);
+    movl(result, cnt2);
+    andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
     jccb(Assembler::zero, COMPARE_TAIL);
+    lea(str2, Address(str2, cnt2, Address::times_2));
+    lea(str1, Address(str1, cnt2, Address::times_2));
+    negptr(cnt2);
+    bind(COMPARE_VECTORS);
+    movdqu(vec1, Address(str1, cnt2, Address::times_2));
+    movdqu(vec2, Address(str2, cnt2, Address::times_2));
+    pxor(vec1, vec2);
+    ptest(vec1, vec1);
+    jccb(Assembler::notZero, VECTOR_NOT_EQUAL);
+    addptr(cnt2, 8);
+    jcc(Assembler::notZero, COMPARE_VECTORS);
+    jmpb(COMPARE_TAIL);
+    lea(str1, Address(str1, result, scale));
+    lea(str2, Address(str2, result, scale));
+    negptr(result);
+    // pcmpestri
+    //   inputs:
+    //     vec1- substring
+    //     rax - negative string length (elements count)
+    //     mem - scaned string
+    //     rdx - string length (elements count)
+    //     pcmpmask - cmp mode: 11000 (string compare with negated result)
+    //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
+    //   outputs:
+    //     rcx - first mismatched element index
+    assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
+    bind(COMPARE_WIDE_VECTORS);
+    movdqu(vec1, Address(str1, result, scale));
+    pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
+    // After pcmpestri cnt1(rcx) contains mismatched element index
+    jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
+    addptr(result, stride);
+    subptr(cnt2, stride);
+    jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
+    // compare wide vectors tail
+    testl(result, result);
+    jccb(Assembler::zero, LENGTH_DIFF_LABEL);
+    movl(cnt2, stride);
+    movl(result, stride);
+    negptr(result);
+    movdqu(vec1, Address(str1, result, scale));
+    pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
+    jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
     // Mismatched characters in the vectors
     bind(VECTOR_NOT_EQUAL);
+    lea(str1, Address(str1, cnt2, Address::times_2));
+    lea(str2, Address(str2, cnt2, Address::times_2));
+    movl(cnt1, 8);
+    // Compare tail (< 8 chars), or rescan last vectors to
+    // find 1st mismatched characters
+    bind(COMPARE_TAIL);
+    testl(cnt1, cnt1);
+    jccb(Assembler::zero, LENGTH_DIFF_LABEL);
+    movl(cnt2, cnt1);
+    addptr(result, cnt1);
+    movptr(cnt2, result);
+    load_unsigned_short(result, Address(str1, cnt2, scale));
+    load_unsigned_short(cnt1, Address(str2, cnt2, scale));
+    subl(result, cnt1);
+    jmpb(POP_LABEL);
+    bind(COMPARE_TAIL); // limit is zero
+    movl(cnt2, result);
     // Fallthru to tail compare
+  }
   // Shift str2 and str1 to the end of the arrays, negate min
   lea(str1, Address(str1, cnt2, Address::times_2, 0));
   lea(str2, Address(str2, cnt2, Address::times_2, 0));
+  lea(str1, Address(str1, cnt2, scale, 0));
+  lea(str2, Address(str2, cnt2, scale, 0));
   negptr(cnt2);
     // Compare the rest of the characters
+  // Compare the rest of the elements
   bind(WHILE_HEAD_LABEL);
   load_unsigned_short(result, Address(str1, cnt2, Address::times_2, 0));
   load_unsigned_short(cnt1, Address(str2, cnt2, Address::times_2, 0));
+  load_unsigned_short(result, Address(str1, cnt2, scale, 0));
+  load_unsigned_short(cnt1, Address(str2, cnt2, scale, 0));
   subl(result, cnt1);
   jccb(Assembler::notZero, POP_LABEL);
   increment(cnt2);
   jcc(Assembler::notZero, WHILE_HEAD_LABEL);
+  jccb(Assembler::notZero, WHILE_HEAD_LABEL);
   // Strings are equal up to min length.  Return the length difference.
 …
   // Discard the stored length difference
   bind(POP_LABEL);
   addptr(rsp, wordSize);
+  pop(cnt1);
   // That's it
 …
     // With SSE4.2, use double quad vector compare
     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
     // Compare 16-byte vectors
     andl(result, 0x0000000e);  //   tail count (in bytes)
 …
     movdqu(vec2, Address(ary2, limit, Address::times_1));
     pxor(vec1, vec2);
     ptest(vec1, vec1);
     jccb(Assembler::notZero, FALSE_LABEL);
     addptr(limit, 16);
     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
+    testl(result, result);
+    jccb(Assembler::zero, TRUE_LABEL);
+    movdqu(vec1, Address(ary1, result, Address::times_1, -16));
+    movdqu(vec2, Address(ary2, result, Address::times_1, -16));
+    pxor(vec1, vec2);
+    ptest(vec1, vec1);
+    jccb(Assembler::notZero, FALSE_LABEL);
+    jmpb(TRUE_LABEL);
     bind(COMPARE_TAIL); // limit is zero

trunk/openjdk/hotspot/src/cpu/x86/vm/assembler_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_ASSEMBLER_X86_HPP
+#define CPU_X86_VM_ASSEMBLER_X86_HPP
 class BiasedLockingCounters;
 …
 // will cause an assertion failure
 #define rscratch1 noreg
+#define rscratch2 noreg
 #endif // _LP64
 …
 #ifdef _LP64
+ static bool is_simm(int64_t x, int nbits) { return -( CONST64(1) << (nbits-1) )  <= x   &&   x  <  ( CONST64(1) << (nbits-1) ); }
+ static bool is_simm(int64_t x, int nbits) { return -(CONST64(1) << (nbits-1)) <= x &&
+                                                    x < (CONST64(1) << (nbits-1)); }
  static bool is_simm32(int64_t x) { return x == (int64_t)(int32_t)x; }
 #else
+ static bool is_simm(int32_t x, int nbits) { return -( 1 << (nbits-1) )  <= x   &&   x  <  ( 1 << (nbits-1) ); }
+ static bool is_simm(int32_t x, int nbits) { return -(1 << (nbits-1)) <= x &&
+                                                    x < (1 << (nbits-1)); }
  static bool is_simm32(int32_t x) { return true; }
 #endif // LP64
+#endif // _LP64
   // Generic instructions
 …
   void push(void* v);
   void pop(void* v);
   // These do register sized moves/scans
 …
   // Vanilla instructions in lexical order
+  void adcl(Address dst, int32_t imm32);
+  void adcl(Address dst, Register src);
   void adcl(Register dst, int32_t imm32);
   void adcl(Register dst, Address src);
 …
   void adcq(Register dst, Address src);
   void adcq(Register dst, Register src);
   void addl(Address dst, int32_t imm32);
 …
   void addq(Register dst, Register src);
   void addr_nop_4();
   void addr_nop_5();
 …
   void andq(Register dst, Address src);
   void andq(Register dst, Register src);
   // Bitwise Logical AND of Packed Double-Precision Floating-Point Values
 …
   void idivl(Register src);
+  void divl(Register src); // Unsigned division
   void idivq(Register src);
 …
   void movq(Register dst, Register src);
   void movq(Register dst, Address src);
   void movq(Address dst, Register src);
+  void movq(Address  dst, Register src);
 #endif
 …
   // Move signed 32bit immediate to 64bit extending sign
   void movslq(Address dst, int32_t imm64);
+  void movslq(Address  dst, int32_t imm64);
   void movslq(Register dst, int32_t imm64);
 …
   void prefetchw(Address src);
+  // POR - Bitwise logical OR
+  void por(XMMRegister dst, XMMRegister src);
   // Shuffle Packed Doublewords
   void pshufd(XMMRegister dst, XMMRegister src, int mode);
 …
   void sqrtsd(XMMRegister dst, Address src);
   void sqrtsd(XMMRegister dst, XMMRegister src);
+  // Compute Square Root of Scalar Single-Precision Floating-Point Value
+  void sqrtss(XMMRegister dst, Address src);
+  void sqrtss(XMMRegister dst, XMMRegister src);
   void std() { emit_byte(0xfd); }
 …
   void store_klass(Register dst, Register src);
+  void load_heap_oop(Register dst, Address src);
+  void store_heap_oop(Address dst, Register src);
+  // Used for storing NULL. All other oop constants should be
+  // stored using routines that take a jobject.
+  void store_heap_oop_null(Address dst);
   void load_prototype_header(Register dst, Register src);
 #ifdef _LP64
   void store_klass_gap(Register dst, Register src);
-  void load_heap_oop(Register dst, Address src);
-  void store_heap_oop(Address dst, Register src);
   // This dummy is to prevent a call to store_heap_oop from
 …
   void store_heap_oop(Address dst, void* dummy);
-  // Used for storing NULL. All other oop constants should be
-  // stored using routines that take a jobject.
-  void store_heap_oop_null(Address dst);
   void encode_heap_oop(Register r);
 …
     Label&   slow_case                 // continuation point if fast allocation fails
   );
+  void tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case);
+  Register tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case); // returns TLS address
+  void incr_allocated_bytes(Register thread,
+                            Register var_size_in_bytes, int con_size_in_bytes,
+                            Register t1 = noreg);
   // interface method calling
 …
   void untested()                                { stop("untested"); }
   void unimplemented(const char* what = "")      { char* b = new char[1024];  jio_snprintf(b, sizeof(b), "unimplemented: %s", what);  stop(b); }
+  void unimplemented(const char* what = "")      { char* b = new char[1024];  jio_snprintf(b, 1024, "unimplemented: %s", what);  stop(b); }
   void should_not_reach_here()                   { stop("should not reach here"); }
 …
   void comisd(XMMRegister dst, AddressLiteral src);
+  void fadd_s(Address src)        { Assembler::fadd_s(src); }
+  void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
   void fldcw(Address src) { Assembler::fldcw(src); }
   void fldcw(AddressLiteral src);
 …
   void fld_x(Address src) { Assembler::fld_x(src); }
   void fld_x(AddressLiteral src);
+  void fmul_s(Address src)        { Assembler::fmul_s(src); }
+  void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
   void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
 …
 public:
+  void addsd(XMMRegister dst, XMMRegister src)    { Assembler::addsd(dst, src); }
+  void addsd(XMMRegister dst, Address src)        { Assembler::addsd(dst, src); }
+  void addsd(XMMRegister dst, AddressLiteral src) { Assembler::addsd(dst, as_Address(src)); }
+  void addss(XMMRegister dst, XMMRegister src)    { Assembler::addss(dst, src); }
+  void addss(XMMRegister dst, Address src)        { Assembler::addss(dst, src); }
+  void addss(XMMRegister dst, AddressLiteral src) { Assembler::addss(dst, as_Address(src)); }
+  void divsd(XMMRegister dst, XMMRegister src)    { Assembler::divsd(dst, src); }
+  void divsd(XMMRegister dst, Address src)        { Assembler::divsd(dst, src); }
+  void divsd(XMMRegister dst, AddressLiteral src) { Assembler::divsd(dst, as_Address(src)); }
+  void divss(XMMRegister dst, XMMRegister src)    { Assembler::divss(dst, src); }
+  void divss(XMMRegister dst, Address src)        { Assembler::divss(dst, src); }
+  void divss(XMMRegister dst, AddressLiteral src) { Assembler::divss(dst, as_Address(src)); }
   void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); }
   void movsd(Address dst, XMMRegister src)     { Assembler::movsd(dst, src); }
   void movsd(XMMRegister dst, Address src)     { Assembler::movsd(dst, src); }
+  void movsd(XMMRegister dst, AddressLiteral src);
+  void movsd(XMMRegister dst, AddressLiteral src) { Assembler::movsd(dst, as_Address(src)); }
+  void mulsd(XMMRegister dst, XMMRegister src)    { Assembler::mulsd(dst, src); }
+  void mulsd(XMMRegister dst, Address src)        { Assembler::mulsd(dst, src); }
+  void mulsd(XMMRegister dst, AddressLiteral src) { Assembler::mulsd(dst, as_Address(src)); }
+  void mulss(XMMRegister dst, XMMRegister src)    { Assembler::mulss(dst, src); }
+  void mulss(XMMRegister dst, Address src)        { Assembler::mulss(dst, src); }
+  void mulss(XMMRegister dst, AddressLiteral src) { Assembler::mulss(dst, as_Address(src)); }
+  void sqrtsd(XMMRegister dst, XMMRegister src)    { Assembler::sqrtsd(dst, src); }
+  void sqrtsd(XMMRegister dst, Address src)        { Assembler::sqrtsd(dst, src); }
+  void sqrtsd(XMMRegister dst, AddressLiteral src) { Assembler::sqrtsd(dst, as_Address(src)); }
+  void sqrtss(XMMRegister dst, XMMRegister src)    { Assembler::sqrtss(dst, src); }
+  void sqrtss(XMMRegister dst, Address src)        { Assembler::sqrtss(dst, src); }
+  void sqrtss(XMMRegister dst, AddressLiteral src) { Assembler::sqrtss(dst, as_Address(src)); }
+  void subsd(XMMRegister dst, XMMRegister src)    { Assembler::subsd(dst, src); }
+  void subsd(XMMRegister dst, Address src)        { Assembler::subsd(dst, src); }
+  void subsd(XMMRegister dst, AddressLiteral src) { Assembler::subsd(dst, as_Address(src)); }
+  void subss(XMMRegister dst, XMMRegister src)    { Assembler::subss(dst, src); }
+  void subss(XMMRegister dst, Address src)        { Assembler::subss(dst, src); }
+  void subss(XMMRegister dst, AddressLiteral src) { Assembler::subss(dst, as_Address(src)); }
   void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
 …
   void string_compare(Register str1, Register str2,
                       Register cnt1, Register cnt2, Register result,
                       XMMRegister vec1, XMMRegister vec2);
+                      XMMRegister vec1);
   // Compare char[] arrays.
 …
 inline bool AbstractAssembler::pd_check_instruction_mark() { return true; }
 #endif
+#endif // CPU_X86_VM_ASSEMBLER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/assembler_x86.inline.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_ASSEMBLER_X86_INLINE_HPP
+#define CPU_X86_VM_ASSEMBLER_X86_INLINE_HPP
+#include "asm/assembler.inline.hpp"
+#include "asm/codeBuffer.hpp"
+#include "code/codeCache.hpp"
+#include "runtime/handles.inline.hpp"
 inline void MacroAssembler::pd_patch_instruction(address branch, address target) {
 …
+}
 #endif // _LP64
+#endif // CPU_X86_VM_ASSEMBLER_X86_INLINE_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/bytecodeInterpreter_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_bytecodeInterpreter_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "interpreter/bytecodeInterpreter.hpp"
+#include "interpreter/bytecodeInterpreter.inline.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
+#ifdef TARGET_ARCH_MODEL_x86_32
+# include "interp_masm_x86_32.hpp"
+#endif
+#ifdef TARGET_ARCH_MODEL_x86_64
+# include "interp_masm_x86_64.hpp"
+#endif
 #ifdef CC_INTERP

trunk/openjdk/hotspot/src/cpu/x86/vm/bytecodeInterpreter_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_BYTECODEINTERPRETER_X86_HPP
+#define CPU_X86_VM_BYTECODEINTERPRETER_X86_HPP
 // Platform specific for C++ based Interpreter
 …
 #define SET_LOCALS_LONG_FROM_ADDR(addr, offset) (((VMJavaVal64*)&locals[-((offset)+1)])->l = \
                                                 ((VMJavaVal64*)(addr))->l)
+#endif // CPU_X86_VM_BYTECODEINTERPRETER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/bytecodeInterpreter_x86.inline.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2002, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_BYTECODEINTERPRETER_X86_INLINE_HPP
+#define CPU_X86_VM_BYTECODEINTERPRETER_X86_INLINE_HPP
 // Inline interpreter functions for IA32
 …
   return (jbyte) val;
+}
+#endif // CPU_X86_VM_BYTECODEINTERPRETER_X86_INLINE_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/bytecodes_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
 #include "incls/_precompiled.incl"
 #include "incls/_bytecodes_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/bytecodes.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/bytecodes_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_BYTECODES_X86_HPP
+#define CPU_X86_VM_BYTECODES_X86_HPP
 // No i486 specific bytecodes
+#endif // CPU_X86_VM_BYTECODES_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/bytes_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2001, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_BYTES_X86_HPP
+#define CPU_X86_VM_BYTES_X86_HPP
+#include "memory/allocation.hpp"
 class Bytes: AllStatic {
 …
 // The following header contains the implementations of swap_u2, swap_u4, and swap_u8[_base]
+#include "incls/_bytes_pd.inline.hpp.incl"
+#ifdef TARGET_OS_ARCH_linux_x86
+# include "bytes_linux_x86.inline.hpp"
+#endif
+#ifdef TARGET_OS_ARCH_solaris_x86
+# include "bytes_solaris_x86.inline.hpp"
+#endif
+#ifdef TARGET_OS_ARCH_windows_x86
+# include "bytes_windows_x86.inline.hpp"
+#endif
+#endif // CPU_X86_VM_BYTES_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_CodeStubs_x86.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_c1_CodeStubs_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_CodeStubs.hpp"
+#include "c1/c1_FrameMap.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "c1/c1_Runtime1.hpp"
+#include "nativeInst_x86.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifndef SERIALGC
+#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
+#endif
 …
+}
-#ifdef TIERED
 void CounterOverflowStub::emit_code(LIR_Assembler* ce) {
   __ bind(_entry);
+  ce->store_parameter(_method->as_register(), 1);
   ce->store_parameter(_bci, 0);
   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::counter_overflow_id)));
   ce->add_call_info_here(_info);
   ce->verify_oop_map(_info);
+  __ jmp(_continuation);
+}
+#endif // TIERED
+  __ jmp(_continuation);
+}
 RangeCheckStub::RangeCheckStub(CodeEmitInfo* info, LIR_Opr index,
 …
   , _index(index)
+{
+  _info = info == NULL ? NULL : new CodeEmitInfo(info);
+  assert(info != NULL, "must have info");
+  _info = new CodeEmitInfo(info);
+}
 …
   Register pre_val_reg = pre_val()->as_register();
   ce->mem2reg(addr(), pre_val(), T_OBJECT, patch_code(), info(), false);
+  ce->mem2reg(addr(), pre_val(), T_OBJECT, patch_code(), info(), false /*wide*/, false /*unaligned*/);
   __ cmpptr(pre_val_reg, (int32_t) NULL_WORD);
 …
   __ cmpptr(new_val_reg, (int32_t) NULL_WORD);
   __ jcc(Assembler::equal, _continuation);
   ce->store_parameter(addr()->as_register(), 0);
+  ce->store_parameter(addr()->as_pointer_register(), 0);
   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id)));
   __ jmp(_continuation);

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_Defs_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_C1_DEFS_X86_HPP
+#define CPU_X86_VM_C1_DEFS_X86_HPP
 // native word offsets from memory address (little endian)
 …
   pd_first_cpu_reg = 0,
   pd_last_cpu_reg = NOT_LP64(5) LP64_ONLY(11),
   pd_first_byte_reg = 2,
   pd_last_byte_reg = 5,
+  pd_first_byte_reg = NOT_LP64(2) LP64_ONLY(0),
+  pd_last_byte_reg = NOT_LP64(5) LP64_ONLY(11),
   pd_first_fpu_reg = pd_nof_cpu_regs_frame_map,
   pd_last_fpu_reg =  pd_first_fpu_reg + 7,
 …
   pd_float_saved_as_double = true
 };
+#endif // CPU_X86_VM_C1_DEFS_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_FpuStackSim_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_c1_FpuStackSim_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_FpuStackSim.hpp"
+#include "c1/c1_FrameMap.hpp"
+#include "utilities/array.hpp"
+#include "utilities/ostream.hpp"
 //--------------------------------------------------------

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_FpuStackSim_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_C1_FPUSTACKSIM_X86_HPP
+#define CPU_X86_VM_C1_FPUSTACKSIM_X86_HPP
 //  Simulates the FPU stack and maintains mapping [fpu-register -> stack offset]
 …
   void print() PRODUCT_RETURN;
 };
+#endif // CPU_X86_VM_C1_FPUSTACKSIM_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_FrameMap_x86.cpp

-              r278
+              r309
  */
+# include "incls/_precompiled.incl"
+# include "incls/_c1_FrameMap_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_FrameMap.hpp"
+#include "c1/c1_LIR.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "vmreg_x86.inline.hpp"
 const int FrameMap::pd_c_runtime_reserved_arg_size = 0;
 …
   map_register( 7, r9);    r9_opr = LIR_OprFact::single_cpu(7);
   map_register( 8, r11);  r11_opr = LIR_OprFact::single_cpu(8);
+  map_register( 9, r12);  r12_opr = LIR_OprFact::single_cpu(9);
+  map_register(10, r13);  r13_opr = LIR_OprFact::single_cpu(10);
+  map_register(11, r14);  r14_opr = LIR_OprFact::single_cpu(11);
+  map_register( 9, r13);  r13_opr = LIR_OprFact::single_cpu(9);
+  map_register(10, r14);  r14_opr = LIR_OprFact::single_cpu(10);
+  // r12 is allocated conditionally. With compressed oops it holds
+  // the heapbase value and is not visible to the allocator.
+  map_register(11, r12);  r12_opr = LIR_OprFact::single_cpu(11);
   // The unallocatable registers are at the end
   map_register(12, r10);  r10_opr = LIR_OprFact::single_cpu(12);
 …
   _caller_save_cpu_regs[7]  = r9_opr;
   _caller_save_cpu_regs[8]  = r11_opr;
   _caller_save_cpu_regs[9]  = r12_opr;
   _caller_save_cpu_regs[10] = r13_opr;
   _caller_save_cpu_regs[11] = r14_opr;
+  _caller_save_cpu_regs[9]  = r13_opr;
+  _caller_save_cpu_regs[10] = r14_opr;
+  _caller_save_cpu_regs[11] = r12_opr;
 #endif // _LP64

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_FrameMap_x86.hpp

-              r278
+              r309
+ *
  */
+#ifndef CPU_X86_VM_C1_FRAMEMAP_X86_HPP
+#define CPU_X86_VM_C1_FRAMEMAP_X86_HPP
 //  On i486 the frame looks as follows:
 …
     return _caller_save_xmm_regs[i];
+  }
+  static int adjust_reg_range(int range) {
+    // Reduce the number of available regs (to free r12) in case of compressed oops
+    if (UseCompressedOops) return range - 1;
+    return range;
+  }
+  static int nof_caller_save_cpu_regs() { return adjust_reg_range(pd_nof_caller_save_cpu_regs_frame_map); }
+  static int last_cpu_reg()             { return adjust_reg_range(pd_last_cpu_reg);  }
+  static int last_byte_reg()            { return adjust_reg_range(pd_last_byte_reg); }
+#endif // CPU_X86_VM_C1_FRAMEMAP_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_c1_LIRAssembler_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_Compilation.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "c1/c1_Runtime1.hpp"
+#include "c1/c1_ValueStack.hpp"
+#include "ci/ciArrayKlass.hpp"
+#include "ci/ciInstance.hpp"
+#include "gc_interface/collectedHeap.hpp"
+#include "memory/barrierSet.hpp"
+#include "memory/cardTableModRefBS.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "runtime/sharedRuntime.hpp"
 …
   Register ic_klass = IC_Klass;
   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
   if (!VerifyOops) {
+  const bool do_post_padding = VerifyOops || UseCompressedOops;
+  if (!do_post_padding) {
     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
     while ((__ offset() + ic_cmp_size) % CodeEntryAlignment != 0) {
 …
   int offset = __ offset();
   __ inline_cache_check(receiver, IC_Klass);
   assert(__ offset() % CodeEntryAlignment == 0 || VerifyOops, "alignment must be correct");
   if (VerifyOops) {
+  assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
+  if (do_post_padding) {
     // force alignment after the cache check.
     // It's been verified to be aligned if !VerifyOops
 …
   // Get addresses of first characters from both Strings
   __ movptr (rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
   __ movptr (rcx, Address(rax, java_lang_String::offset_offset_in_bytes()));
   __ lea    (rsi, Address(rsi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
+  __ load_heap_oop(rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
+  __ movptr       (rcx, Address(rax, java_lang_String::offset_offset_in_bytes()));
+  __ lea          (rsi, Address(rsi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
   // rbx, may be NULL
   add_debug_info_for_null_check_here(info);
   __ movptr (rdi, Address(rbx, java_lang_String::value_offset_in_bytes()));
   __ movptr (rcx, Address(rbx, java_lang_String::offset_offset_in_bytes()));
   __ lea    (rdi, Address(rdi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
+  __ load_heap_oop(rdi, Address(rbx, java_lang_String::value_offset_in_bytes()));
+  __ movptr       (rcx, Address(rbx, java_lang_String::offset_offset_in_bytes()));
+  __ lea          (rdi, Address(rdi, rcx, Address::times_2, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
   // compute minimum length (in rax) and difference of lengths (on top of stack)
 …
   switch (c->type()) {
+    case T_INT:
+    case T_INT: {
+      assert(patch_code == lir_patch_none, "no patching handled here");
+      __ movl(dest->as_register(), c->as_jint());
+      break;
+    }
     case T_ADDRESS: {
       assert(patch_code == lir_patch_none, "no patching handled here");
       __ movl(dest->as_register(), c->as_jint());
+      __ movptr(dest->as_register(), c->as_jint());
       break;
+    }
 …
     case T_INT:  // fall through
     case T_FLOAT:
+      __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
+      break;
     case T_ADDRESS:
       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
+      __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
       break;
 …
+}
 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info ) {
+void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
   assert(src->is_constant(), "should not call otherwise");
   assert(dest->is_address(), "should not call otherwise");
 …
     case T_INT:    // fall through
     case T_FLOAT:
+      __ movl(as_Address(addr), c->as_jint_bits());
+      break;
     case T_ADDRESS:
       __ movl(as_Address(addr), c->as_jint_bits());
+      __ movptr(as_Address(addr), c->as_jint_bits());
       break;
 …
     case T_ARRAY:
       if (c->as_jobject() == NULL) {
+        __ movptr(as_Address(addr), NULL_WORD);
+        if (UseCompressedOops && !wide) {
+          __ movl(as_Address(addr), (int32_t)NULL_WORD);
+        } else {
+          __ movptr(as_Address(addr), NULL_WORD);
+        }
       } else {
         if (is_literal_address(addr)) {
 …
 #ifdef _LP64
           __ movoop(rscratch1, c->as_jobject());
+          null_check_here = code_offset();
+          __ movptr(as_Address_lo(addr), rscratch1);
+          if (UseCompressedOops && !wide) {
+            __ encode_heap_oop(rscratch1);
+            null_check_here = code_offset();
+            __ movl(as_Address_lo(addr), rscratch1);
+          } else {
+            null_check_here = code_offset();
+            __ movptr(as_Address_lo(addr), rscratch1);
+          }
 #else
           __ movoop(as_Address(addr), c->as_jobject());
 …
 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool /* unaligned */) {
+void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
   LIR_Address* to_addr = dest->as_address_ptr();
   PatchingStub* patch = NULL;
+  Register compressed_src = rscratch1;
   if (type == T_ARRAY || type == T_OBJECT) {
     __ verify_oop(src->as_register());
+  }
+#ifdef _LP64
+    if (UseCompressedOops && !wide) {
+      __ movptr(compressed_src, src->as_register());
+      __ encode_heap_oop(compressed_src);
+    }
+#endif
+  }
   if (patch_code != lir_patch_none) {
     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 …
     assert(toa.disp() != 0, "must have");
+  }
+  if (info != NULL) {
+    add_debug_info_for_null_check_here(info);
+  }
+  int null_check_here = code_offset();
   switch (type) {
     case T_FLOAT: {
 …
+    }
-    case T_ADDRESS: // fall through
     case T_ARRAY:   // fall through
     case T_OBJECT:  // fall through
+#ifdef _LP64
+      if (UseCompressedOops && !wide) {
+        __ movl(as_Address(to_addr), compressed_src);
+      } else {
+        __ movptr(as_Address(to_addr), src->as_register());
+      }
+      break;
+    case T_ADDRESS:
       __ movptr(as_Address(to_addr), src->as_register());
       break;
-#endif // _LP64
     case T_INT:
       __ movl(as_Address(to_addr), src->as_register());
 …
       ShouldNotReachHere();
+  }
+  if (info != NULL) {
+    add_debug_info_for_null_check(null_check_here, info);
+  }
   if (patch_code != lir_patch_none) {
 …
 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool /* unaligned */) {
+void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
   assert(src->is_address(), "should not call otherwise");
   assert(dest->is_register(), "should not call otherwise");
 …
+    }
-    case T_ADDRESS: // fall through
     case T_OBJECT:  // fall through
     case T_ARRAY:   // fall through
+#ifdef _LP64
+      if (UseCompressedOops && !wide) {
+        __ movl(dest->as_register(), from_addr);
+      } else {
+        __ movptr(dest->as_register(), from_addr);
+      }
+      break;
+    case T_ADDRESS:
       __ movptr(dest->as_register(), from_addr);
       break;
-#endif // _L64
     case T_INT:
       __ movl(dest->as_register(), from_addr);
 …
   if (type == T_ARRAY || type == T_OBJECT) {
+#ifdef _LP64
+    if (UseCompressedOops && !wide) {
+      __ decode_heap_oop(dest->as_register());
+    }
+#endif
     __ verify_oop(dest->as_register());
+  }
 …
         ShouldNotReachHere(); break;
+    }
   } else if (VM_Version::supports_3dnow()) {
+  } else if (VM_Version::supports_3dnow_prefetch()) {
     __ prefetchr(from_addr);
+  }
 …
         ShouldNotReachHere(); break;
+    }
   } else if (VM_Version::supports_3dnow()) {
+  } else if (VM_Version::supports_3dnow_prefetch()) {
     __ prefetchw(from_addr);
+  }
 …
 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
+  Register len =  op->len()->as_register();
+  LP64_ONLY( __ movslq(len, len); )
   if (UseSlowPath ||
       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
 …
     __ jmp(*op->stub()->entry());
   } else {
-    Register len =  op->len()->as_register();
     Register tmp1 = op->tmp1()->as_register();
     Register tmp2 = op->tmp2()->as_register();
 …
+}
+void LIR_Assembler::type_profile_helper(Register mdo,
+                                        ciMethodData *md, ciProfileData *data,
+                                        Register recv, Label* update_done) {
+  for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
+    Label next_test;
+    // See if the receiver is receiver[n].
+    __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
+    __ jccb(Assembler::notEqual, next_test);
+    Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
+    __ addptr(data_addr, DataLayout::counter_increment);
+    __ jmp(*update_done);
+    __ bind(next_test);
+  }
+  // Didn't find receiver; find next empty slot and fill it in
+  for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
+    Label next_test;
+    Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
+    __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
+    __ jccb(Assembler::notEqual, next_test);
+    __ movptr(recv_addr, recv);
+    __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
+    __ jmp(*update_done);
+    __ bind(next_test);
+  }
+}
+void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
+  // we always need a stub for the failure case.
+  CodeStub* stub = op->stub();
+  Register obj = op->object()->as_register();
+  Register k_RInfo = op->tmp1()->as_register();
+  Register klass_RInfo = op->tmp2()->as_register();
+  Register dst = op->result_opr()->as_register();
+  ciKlass* k = op->klass();
+  Register Rtmp1 = noreg;
+  // check if it needs to be profiled
+  ciMethodData* md;
+  ciProfileData* data;
+  if (op->should_profile()) {
+    ciMethod* method = op->profiled_method();
+    assert(method != NULL, "Should have method");
+    int bci = op->profiled_bci();
+    md = method->method_data_or_null();
+    assert(md != NULL, "Sanity");
+    data = md->bci_to_data(bci);
+    assert(data != NULL,                "need data for type check");
+    assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
+  }
+  Label profile_cast_success, profile_cast_failure;
+  Label *success_target = op->should_profile() ? &profile_cast_success : success;
+  Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
+  if (obj == k_RInfo) {
+    k_RInfo = dst;
+  } else if (obj == klass_RInfo) {
+    klass_RInfo = dst;
+  }
+  if (k->is_loaded() && !UseCompressedOops) {
+    select_different_registers(obj, dst, k_RInfo, klass_RInfo);
+  } else {
+    Rtmp1 = op->tmp3()->as_register();
+    select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
+  }
+  assert_different_registers(obj, k_RInfo, klass_RInfo);
+  if (!k->is_loaded()) {
+    jobject2reg_with_patching(k_RInfo, op->info_for_patch());
+  } else {
+#ifdef _LP64
+    __ movoop(k_RInfo, k->constant_encoding());
+#endif // _LP64
+  }
+  assert(obj != k_RInfo, "must be different");
+  __ cmpptr(obj, (int32_t)NULL_WORD);
+  if (op->should_profile()) {
+    Label not_null;
+    __ jccb(Assembler::notEqual, not_null);
+    // Object is null; update MDO and exit
+    Register mdo  = klass_RInfo;
+    __ movoop(mdo, md->constant_encoding());
+    Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
+    int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
+    __ orl(data_addr, header_bits);
+    __ jmp(*obj_is_null);
+    __ bind(not_null);
+  } else {
+    __ jcc(Assembler::equal, *obj_is_null);
+  }
+  __ verify_oop(obj);
+  if (op->fast_check()) {
+    // get object class
+    // not a safepoint as obj null check happens earlier
+#ifdef _LP64
+    if (UseCompressedOops) {
+      __ load_klass(Rtmp1, obj);
+      __ cmpptr(k_RInfo, Rtmp1);
+    } else {
+      __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+    }
+#else
+    if (k->is_loaded()) {
+      __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
+    } else {
+      __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+    }
+#endif
+    __ jcc(Assembler::notEqual, *failure_target);
+    // successful cast, fall through to profile or jump
+  } else {
+    // get object class
+    // not a safepoint as obj null check happens earlier
+    __ load_klass(klass_RInfo, obj);
+    if (k->is_loaded()) {
+      // See if we get an immediate positive hit
+#ifdef _LP64
+      __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
+#else
+      __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
+#endif // _LP64
+      if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() != k->super_check_offset()) {
+        __ jcc(Assembler::notEqual, *failure_target);
+        // successful cast, fall through to profile or jump
+      } else {
+        // See if we get an immediate positive hit
+        __ jcc(Assembler::equal, *success_target);
+        // check for self
+#ifdef _LP64
+        __ cmpptr(klass_RInfo, k_RInfo);
+#else
+        __ cmpoop(klass_RInfo, k->constant_encoding());
+#endif // _LP64
+        __ jcc(Assembler::equal, *success_target);
+        __ push(klass_RInfo);
+#ifdef _LP64
+        __ push(k_RInfo);
+#else
+        __ pushoop(k->constant_encoding());
+#endif // _LP64
+        __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
+        __ pop(klass_RInfo);
+        __ pop(klass_RInfo);
+        // result is a boolean
+        __ cmpl(klass_RInfo, 0);
+        __ jcc(Assembler::equal, *failure_target);
+        // successful cast, fall through to profile or jump
+      }
+    } else {
+      // perform the fast part of the checking logic
+      __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
+      // call out-of-line instance of __ check_klass_subtype_slow_path(...):
+      __ push(klass_RInfo);
+      __ push(k_RInfo);
+      __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
+      __ pop(klass_RInfo);
+      __ pop(k_RInfo);
+      // result is a boolean
+      __ cmpl(k_RInfo, 0);
+      __ jcc(Assembler::equal, *failure_target);
+      // successful cast, fall through to profile or jump
+    }
+  }
+  if (op->should_profile()) {
+    Register mdo  = klass_RInfo, recv = k_RInfo;
+    __ bind(profile_cast_success);
+    __ movoop(mdo, md->constant_encoding());
+    __ load_klass(recv, obj);
+    Label update_done;
+    type_profile_helper(mdo, md, data, recv, success);
+    __ jmp(*success);
+    __ bind(profile_cast_failure);
+    __ movoop(mdo, md->constant_encoding());
+    Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
+    __ subptr(counter_addr, DataLayout::counter_increment);
+    __ jmp(*failure);
+  }
+  __ jmp(*success);
+}
 …
     CodeStub* stub = op->stub();
+    Label done;
+    // check if it needs to be profiled
+    ciMethodData* md;
+    ciProfileData* data;
+    if (op->should_profile()) {
+      ciMethod* method = op->profiled_method();
+      assert(method != NULL, "Should have method");
+      int bci = op->profiled_bci();
+      md = method->method_data_or_null();
+      assert(md != NULL, "Sanity");
+      data = md->bci_to_data(bci);
+      assert(data != NULL,                "need data for type check");
+      assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
+    }
+    Label profile_cast_success, profile_cast_failure, done;
+    Label *success_target = op->should_profile() ? &profile_cast_success : &done;
+    Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
     __ cmpptr(value, (int32_t)NULL_WORD);
+    __ jcc(Assembler::equal, done);
+    if (op->should_profile()) {
+      Label not_null;
+      __ jccb(Assembler::notEqual, not_null);
+      // Object is null; update MDO and exit
+      Register mdo  = klass_RInfo;
+      __ movoop(mdo, md->constant_encoding());
+      Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
+      int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
+      __ orl(data_addr, header_bits);
+      __ jmp(done);
+      __ bind(not_null);
+    } else {
+      __ jcc(Assembler::equal, done);
+    }
     add_debug_info_for_null_check_here(op->info_for_exception());
     __ movptr(k_RInfo, Address(array, oopDesc::klass_offset_in_bytes()));
     __ movptr(klass_RInfo, Address(value, oopDesc::klass_offset_in_bytes()));
     // get instance klass
+    __ load_klass(k_RInfo, array);
+    __ load_klass(klass_RInfo, value);
+    // get instance klass (it's already uncompressed)
     __ movptr(k_RInfo, Address(k_RInfo, objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc)));
     // perform the fast part of the checking logic
     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, &done, stub->entry(), NULL);
+    __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
     __ push(klass_RInfo);
 …
     // result is a boolean
     __ cmpl(k_RInfo, 0);
+    __ jcc(Assembler::equal, *stub->entry());
+    __ jcc(Assembler::equal, *failure_target);
+    // fall through to the success case
+    if (op->should_profile()) {
+      Register mdo  = klass_RInfo, recv = k_RInfo;
+      __ bind(profile_cast_success);
+      __ movoop(mdo, md->constant_encoding());
+      __ load_klass(recv, value);
+      Label update_done;
+      type_profile_helper(mdo, md, data, recv, &done);
+      __ jmpb(done);
+      __ bind(profile_cast_failure);
+      __ movoop(mdo, md->constant_encoding());
+      Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
+      __ subptr(counter_addr, DataLayout::counter_increment);
+      __ jmp(*stub->entry());
+    }
     __ bind(done);
+  } else if (op->code() == lir_checkcast) {
+    // we always need a stub for the failure case.
+    CodeStub* stub = op->stub();
+    Register obj = op->object()->as_register();
+    Register k_RInfo = op->tmp1()->as_register();
+    Register klass_RInfo = op->tmp2()->as_register();
+    Register dst = op->result_opr()->as_register();
+    ciKlass* k = op->klass();
+    Register Rtmp1 = noreg;
+    Label done;
+    if (obj == k_RInfo) {
+      k_RInfo = dst;
+    } else if (obj == klass_RInfo) {
+      klass_RInfo = dst;
+    }
+    if (k->is_loaded()) {
+      select_different_registers(obj, dst, k_RInfo, klass_RInfo);
+    } else {
+      Rtmp1 = op->tmp3()->as_register();
+      select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
+    }
+    assert_different_registers(obj, k_RInfo, klass_RInfo);
+    if (!k->is_loaded()) {
+      jobject2reg_with_patching(k_RInfo, op->info_for_patch());
+    } else {
+#ifdef _LP64
+      __ movoop(k_RInfo, k->constant_encoding());
+#else
+      k_RInfo = noreg;
+#endif // _LP64
+    }
+    assert(obj != k_RInfo, "must be different");
+    __ cmpptr(obj, (int32_t)NULL_WORD);
+    if (op->profiled_method() != NULL) {
+      ciMethod* method = op->profiled_method();
+      int bci          = op->profiled_bci();
+      Label profile_done;
+      __ jcc(Assembler::notEqual, profile_done);
+      // Object is null; update methodDataOop
+      ciMethodData* md = method->method_data();
+      if (md == NULL) {
+        bailout("out of memory building methodDataOop");
+        return;
+      }
+      ciProfileData* data = md->bci_to_data(bci);
+      assert(data != NULL,       "need data for checkcast");
+      assert(data->is_BitData(), "need BitData for checkcast");
+      Register mdo  = klass_RInfo;
+      __ movoop(mdo, md->constant_encoding());
+      Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
+      int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
+      __ orl(data_addr, header_bits);
+      __ jmp(done);
+      __ bind(profile_done);
+    } else {
+      __ jcc(Assembler::equal, done);
+    }
+    __ verify_oop(obj);
+    if (op->fast_check()) {
+      // get object classo
+      // not a safepoint as obj null check happens earlier
+      if (k->is_loaded()) {
+#ifdef _LP64
+        __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+#else
+        __ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
+#endif // _LP64
+      } else {
+        __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+      }
+      __ jcc(Assembler::notEqual, *stub->entry());
+      __ bind(done);
+    } else {
+      // get object class
+      // not a safepoint as obj null check happens earlier
+      __ movptr(klass_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+      if (k->is_loaded()) {
+        // See if we get an immediate positive hit
+#ifdef _LP64
+        __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
+#else
+        __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
+#endif // _LP64
+        if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() != k->super_check_offset()) {
+          __ jcc(Assembler::notEqual, *stub->entry());
+        } else {
+          // See if we get an immediate positive hit
+          __ jcc(Assembler::equal, done);
+          // check for self
+#ifdef _LP64
+          __ cmpptr(klass_RInfo, k_RInfo);
+#else
+          __ cmpoop(klass_RInfo, k->constant_encoding());
+#endif // _LP64
+          __ jcc(Assembler::equal, done);
+          __ push(klass_RInfo);
+#ifdef _LP64
+          __ push(k_RInfo);
+#else
+          __ pushoop(k->constant_encoding());
+#endif // _LP64
+          __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
+          __ pop(klass_RInfo);
+          __ pop(klass_RInfo);
+          // result is a boolean
+          __ cmpl(klass_RInfo, 0);
+          __ jcc(Assembler::equal, *stub->entry());
+        }
+  } else
+    if (code == lir_checkcast) {
+      Register obj = op->object()->as_register();
+      Register dst = op->result_opr()->as_register();
+      Label success;
+      emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
+      __ bind(success);
+      if (dst != obj) {
+        __ mov(dst, obj);
+      }
+    } else
+      if (code == lir_instanceof) {
+        Register obj = op->object()->as_register();
+        Register dst = op->result_opr()->as_register();
+        Label success, failure, done;
+        emit_typecheck_helper(op, &success, &failure, &failure);
+        __ bind(failure);
+        __ xorptr(dst, dst);
+        __ jmpb(done);
+        __ bind(success);
+        __ movptr(dst, 1);
         __ bind(done);
       } else {
+        // perform the fast part of the checking logic
+        __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, &done, stub->entry(), NULL);
+        // call out-of-line instance of __ check_klass_subtype_slow_path(...):
+        __ push(klass_RInfo);
+        __ push(k_RInfo);
+        __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
+        __ pop(klass_RInfo);
+        __ pop(k_RInfo);
+        // result is a boolean
+        __ cmpl(k_RInfo, 0);
+        __ jcc(Assembler::equal, *stub->entry());
+        __ bind(done);
+      }
+    }
+    if (dst != obj) {
+      __ mov(dst, obj);
+    }
+  } else if (code == lir_instanceof) {
+    Register obj = op->object()->as_register();
+    Register k_RInfo = op->tmp1()->as_register();
+    Register klass_RInfo = op->tmp2()->as_register();
+    Register dst = op->result_opr()->as_register();
+    ciKlass* k = op->klass();
+    Label done;
+    Label zero;
+    Label one;
+    if (obj == k_RInfo) {
+      k_RInfo = klass_RInfo;
+      klass_RInfo = obj;
+    }
+    // patching may screw with our temporaries on sparc,
+    // so let's do it before loading the class
+    if (!k->is_loaded()) {
+      jobject2reg_with_patching(k_RInfo, op->info_for_patch());
+    } else {
+      LP64_ONLY(__ movoop(k_RInfo, k->constant_encoding()));
+    }
+    assert(obj != k_RInfo, "must be different");
+    __ verify_oop(obj);
+    if (op->fast_check()) {
+      __ cmpptr(obj, (int32_t)NULL_WORD);
+      __ jcc(Assembler::equal, zero);
+      // get object class
+      // not a safepoint as obj null check happens earlier
+      if (LP64_ONLY(false &&) k->is_loaded()) {
+        NOT_LP64(__ cmpoop(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding()));
+        k_RInfo = noreg;
+      } else {
+        __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+      }
+      __ jcc(Assembler::equal, one);
+    } else {
+      // get object class
+      // not a safepoint as obj null check happens earlier
+      __ cmpptr(obj, (int32_t)NULL_WORD);
+      __ jcc(Assembler::equal, zero);
+      __ movptr(klass_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
+#ifndef _LP64
+      if (k->is_loaded()) {
+        // See if we get an immediate positive hit
+        __ cmpoop(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
+        __ jcc(Assembler::equal, one);
+        if (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes() == k->super_check_offset()) {
+          // check for self
+          __ cmpoop(klass_RInfo, k->constant_encoding());
+          __ jcc(Assembler::equal, one);
+          __ push(klass_RInfo);
+          __ pushoop(k->constant_encoding());
+          __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
+          __ pop(klass_RInfo);
+          __ pop(dst);
+          __ jmp(done);
+        }
+      }
+        else // next block is unconditional if LP64:
+#endif // LP64
+      {
+        assert(dst != klass_RInfo && dst != k_RInfo, "need 3 registers");
+        // perform the fast part of the checking logic
+        __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, dst, &one, &zero, NULL);
+        // call out-of-line instance of __ check_klass_subtype_slow_path(...):
+        __ push(klass_RInfo);
+        __ push(k_RInfo);
+        __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
+        __ pop(klass_RInfo);
+        __ pop(dst);
+        __ jmp(done);
+      }
+    }
+    __ bind(zero);
+    __ xorptr(dst, dst);
+    __ jmp(done);
+    __ bind(one);
+    __ movptr(dst, 1);
+    __ bind(done);
+  } else {
+    ShouldNotReachHere();
+  }
+        ShouldNotReachHere();
+      }
+}
 …
     assert(cmpval != addr, "cmp and addr must be in different registers");
     assert(newval != addr, "new value and addr must be in different registers");
+    if (os::is_MP()) {
+      __ lock();
+    }
     if ( op->code() == lir_cas_obj) {
+      __ cmpxchgptr(newval, Address(addr, 0));
+    } else if (op->code() == lir_cas_int) {
+#ifdef _LP64
+      if (UseCompressedOops) {
+        __ encode_heap_oop(cmpval);
+        __ mov(rscratch1, newval);
+        __ encode_heap_oop(rscratch1);
+        if (os::is_MP()) {
+          __ lock();
+        }
+        // cmpval (rax) is implicitly used by this instruction
+        __ cmpxchgl(rscratch1, Address(addr, 0));
+      } else
+#endif
+      {
+        if (os::is_MP()) {
+          __ lock();
+        }
+        __ cmpxchgptr(newval, Address(addr, 0));
+      }
+    } else {
+      assert(op->code() == lir_cas_int, "lir_cas_int expected");
+      if (os::is_MP()) {
+        __ lock();
+      }
       __ cmpxchgl(newval, Address(addr, 0));
-    } else {
-      LP64_ONLY(__ cmpxchgq(newval, Address(addr, 0)));
+    }
 #ifdef _LP64
 …
+}
+void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result) {
+void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
   Assembler::Condition acond, ncond;
   switch (condition) {
 …
       switch (code) {
         case lir_add: {
           __ increment(lreg, c);
+          __ incrementl(lreg, c);
           break;
+        }
         case lir_sub: {
           __ decrement(lreg, c);
+          __ decrementl(lreg, c);
           break;
+        }
 …
   if (flags & LIR_OpArrayCopy::type_check) {
+    __ movptr(tmp, src_klass_addr);
+    __ cmpptr(tmp, dst_klass_addr);
+    if (UseCompressedOops) {
+      __ movl(tmp, src_klass_addr);
+      __ cmpl(tmp, dst_klass_addr);
+    } else {
+      __ movptr(tmp, src_klass_addr);
+      __ cmpptr(tmp, dst_klass_addr);
+    }
     __ jcc(Assembler::notEqual, *stub->entry());
+  }
 …
     Label known_ok, halt;
     __ movoop(tmp, default_type->constant_encoding());
+#ifdef _LP64
+    if (UseCompressedOops) {
+      __ encode_heap_oop(tmp);
+    }
+#endif
     if (basic_type != T_OBJECT) {
+      __ cmpptr(tmp, dst_klass_addr);
+      if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
+      else                   __ cmpptr(tmp, dst_klass_addr);
       __ jcc(Assembler::notEqual, halt);
+      __ cmpptr(tmp, src_klass_addr);
+      if (UseCompressedOops) __ cmpl(tmp, src_klass_addr);
+      else                   __ cmpptr(tmp, src_klass_addr);
       __ jcc(Assembler::equal, known_ok);
     } else {
+      __ cmpptr(tmp, dst_klass_addr);
+      if (UseCompressedOops) __ cmpl(tmp, dst_klass_addr);
+      else                   __ cmpptr(tmp, dst_klass_addr);
       __ jcc(Assembler::equal, known_ok);
       __ cmpptr(src, dst);
 …
   // Update counter for all call types
+  ciMethodData* md = method->method_data();
+  if (md == NULL) {
+    bailout("out of memory building methodDataOop");
+    return;
+  }
+  ciMethodData* md = method->method_data_or_null();
+  assert(md != NULL, "Sanity");
   ciProfileData* data = md->bci_to_data(bci);
   assert(data->is_CounterData(), "need CounterData for calls");
 …
   // invokeinterface bytecodes
   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
       Tier1ProfileVirtualCalls) {
+      C1ProfileVirtualCalls) {
     assert(op->recv()->is_single_cpu(), "recv must be allocated");
     Register recv = op->recv()->as_register();
 …
     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
     ciKlass* known_klass = op->known_holder();
     if (Tier1OptimizeVirtualCallProfiling && known_klass != NULL) {
+    if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
       // We know the type that will be seen at this call site; we can
       // statically update the methodDataOop rather than needing to do
 …
         if (known_klass->equals(receiver)) {
           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
           __ addl(data_addr, DataLayout::counter_increment);
+          __ addptr(data_addr, DataLayout::counter_increment);
           return;
+        }
 …
           __ movoop(recv_addr, known_klass->constant_encoding());
           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
           __ addl(data_addr, DataLayout::counter_increment);
+          __ addptr(data_addr, DataLayout::counter_increment);
           return;
+        }
+      }
     } else {
       __ movptr(recv, Address(recv, oopDesc::klass_offset_in_bytes()));
+      __ load_klass(recv, recv);
       Label update_done;
+      uint i;
+      for (i = 0; i < VirtualCallData::row_limit(); i++) {
+        Label next_test;
+        // See if the receiver is receiver[n].
+        __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))));
+        __ jcc(Assembler::notEqual, next_test);
+        Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
+        __ addl(data_addr, DataLayout::counter_increment);
+        __ jmp(update_done);
+        __ bind(next_test);
+      }
+      // Didn't find receiver; find next empty slot and fill it in
+      for (i = 0; i < VirtualCallData::row_limit(); i++) {
+        Label next_test;
+        Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
+        __ cmpptr(recv_addr, (int32_t)NULL_WORD);
+        __ jcc(Assembler::notEqual, next_test);
+        __ movptr(recv_addr, recv);
+        __ movl(Address(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))), DataLayout::counter_increment);
+        __ jmp(update_done);
+        __ bind(next_test);
+      }
+      type_profile_helper(mdo, md, data, recv, &update_done);
       // Receiver did not match any saved receiver and there is no empty row for it.
       // Increment total counter to indicate polymorphic case.
       __ addl(counter_addr, DataLayout::counter_increment);
+      __ addptr(counter_addr, DataLayout::counter_increment);
       __ bind(update_done);
 …
   } else {
     // Static call
+    __ addl(counter_addr, DataLayout::counter_increment);
+  }
+}
+    __ addptr(counter_addr, DataLayout::counter_increment);
+  }
+}
 void LIR_Assembler::emit_delay(LIR_OpDelay*) {

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_LIRAssembler_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_C1_LIRASSEMBLER_X86_HPP
+#define CPU_X86_VM_C1_LIRASSEMBLER_X86_HPP
  private:
 …
   Address as_Address(LIR_Address* addr, Register tmp);
+  // Record the type of the receiver in ReceiverTypeData
+  void type_profile_helper(Register mdo,
+                           ciMethodData *md, ciProfileData *data,
+                           Register recv, Label* update_done);
 public:
 …
          deopt_handler_size = NOT_LP64(10) LP64_ONLY(17)
        };
+#endif // CPU_X86_VM_C1_LIRASSEMBLER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2005, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_c1_LIRGenerator_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_Compilation.hpp"
+#include "c1/c1_FrameMap.hpp"
+#include "c1/c1_Instruction.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_LIRGenerator.hpp"
+#include "c1/c1_Runtime1.hpp"
+#include "c1/c1_ValueStack.hpp"
+#include "ci/ciArray.hpp"
+#include "ci/ciObjArrayKlass.hpp"
+#include "ci/ciTypeArrayKlass.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "vmreg_x86.inline.hpp"
 #ifdef ASSERT
 …
+  }
   Constant* c = v->as_Constant();
   if (c && c->state() == NULL) {
+  if (c && c->state_before() == NULL) {
     // constants of any type can be stored directly, except for
     // unloaded object constants.
 …
+void LIRGenerator::increment_counter(address counter, int step) {
+LIR_Opr LIRGenerator::load_immediate(int x, BasicType type) {
+  LIR_Opr r;
+  if (type == T_LONG) {
+    r = LIR_OprFact::longConst(x);
+  } else if (type == T_INT) {
+    r = LIR_OprFact::intConst(x);
+  } else {
+    ShouldNotReachHere();
+  }
+  return r;
+}
+void LIRGenerator::increment_counter(address counter, BasicType type, int step) {
   LIR_Opr pointer = new_pointer_register();
   __ move(LIR_OprFact::intptrConst(counter), pointer);
   LIR_Address* addr = new LIR_Address(pointer, T_INT);
+  LIR_Address* addr = new LIR_Address(pointer, type);
   increment_counter(addr, step);
+}
 …
   __ add((LIR_Opr)addr, LIR_OprFact::intConst(step), (LIR_Opr)addr);
+}
 void LIRGenerator::cmp_mem_int(LIR_Condition condition, LIR_Opr base, int disp, int c, CodeEmitInfo* info) {
 …
 void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
   assert(x->is_root(),"");
+  assert(x->is_pinned(),"");
   bool needs_range_check = true;
   bool use_length = x->length() != NULL;
 …
 void LIRGenerator::do_MonitorEnter(MonitorEnter* x) {
   assert(x->is_root(),"");
+  assert(x->is_pinned(),"");
   LIRItem obj(x->obj(), this);
   obj.load_item();
 …
   CodeEmitInfo* info_for_exception = NULL;
   if (x->needs_null_check()) {
     info_for_exception = state_for(x, x->lock_stack_before());
+    info_for_exception = state_for(x);
+  }
   // this CodeEmitInfo must not have the xhandlers because here the
 …
 void LIRGenerator::do_MonitorExit(MonitorExit* x) {
   assert(x->is_root(),"");
+  assert(x->is_pinned(),"");
   LIRItem obj(x->obj(), this);
 …
   // generate compare-and-swap; produces zero condition if swap occurs
   int value_offset = sun_misc_AtomicLongCSImpl::value_offset();
   LIR_Opr addr = obj.result();
   __ add(addr, LIR_OprFact::intConst(value_offset), addr);
+  LIR_Opr addr = new_pointer_register();
+  __ leal(LIR_OprFact::address(new LIR_Address(obj.result(), value_offset, T_LONG)), addr);
   LIR_Opr t1 = LIR_OprFact::illegalOpr;  // no temp needed
   LIR_Opr t2 = LIR_OprFact::illegalOpr;  // no temp needed
 …
   // generate conditional move of boolean result
   LIR_Opr result = rlock_result(x);
   __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), result);
+  __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), result, T_LONG);
+}
 …
   // generate conditional move of boolean result
   LIR_Opr result = rlock_result(x);
+  __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0), result);
+  __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0),
+           result, as_BasicType(type));
   if (type == objectType) {   // Write-barrier needed for Object fields.
     // Seems to be precise
 …
 void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
   assert(x->number_of_arguments() == 5, "wrong type");
+  // Make all state_for calls early since they can emit code
+  CodeEmitInfo* info = state_for(x, x->state());
   LIRItem src(x->argument_at(0), this);
   LIRItem src_pos(x->argument_at(1), this);
 …
   arraycopy_helper(x, &flags, &expected_type);
-  CodeEmitInfo* info = state_for(x, x->state()); // we may want to have stack (deoptimization?)
   __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
+}
 …
 void LIRGenerator::do_NewInstance(NewInstance* x) {
+#ifndef PRODUCT
   if (PrintNotLoaded && !x->klass()->is_loaded()) {
+    tty->print_cr("   ###class not loaded at new bci %d", x->bci());
+  }
+    tty->print_cr("   ###class not loaded at new bci %d", x->printable_bci());
+  }
+#endif
   CodeEmitInfo* info = state_for(x, x->state());
   LIR_Opr reg = result_register_for(x->type());
 …
   // info for exceptions
   CodeEmitInfo* info_for_exception = state_for(x, x->state()->copy_locks());
+  CodeEmitInfo* info_for_exception = state_for(x);
   CodeStub* stub;
 …
+  }
   LIR_Opr reg = rlock_result(x);
+  LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
+  if (!x->klass()->is_loaded() || UseCompressedOops) {
+    tmp3 = new_register(objectType);
+  }
   __ checkcast(reg, obj.result(), x->klass(),
+               new_register(objectType), new_register(objectType),
+               !x->klass()->is_loaded() ? new_register(objectType) : LIR_OprFact::illegalOpr,
+               new_register(objectType), new_register(objectType), tmp3,
                x->direct_compare(), info_for_exception, patching_info, stub,
                x->profiled_method(), x->profiled_bci());
 …
+  }
   obj.load_item();
+  LIR_Opr tmp = new_register(objectType);
+  LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
+  if (!x->klass()->is_loaded() || UseCompressedOops) {
+    tmp3 = new_register(objectType);
+  }
   __ instanceof(reg, obj.result(), x->klass(),
                 tmp, new_register(objectType), LIR_OprFact::illegalOpr,
                 x->direct_compare(), patching_info);
+                new_register(objectType), new_register(objectType), tmp3,
+                x->direct_compare(), patching_info, x->profiled_method(), x->profiled_bci());
+}
 …
   if (x->is_safepoint()) {
     // increment backedge counter if needed
+    increment_backedge_counter(state_for(x, x->state_before()));
+    increment_backedge_counter(state_for(x, x->state_before()), x->profiled_bci());
     __ safepoint(LIR_OprFact::illegalOpr, state_for(x, x->state_before()));
+  }
 …
   LIR_Opr right = yin->result();
   __ cmp(lir_cond(cond), left, right);
+  // Generate branch profiling. Profiling code doesn't kill flags.
   profile_branch(x, cond);
   move_to_phi(x->state());

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_LinearScan_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_c1_LinearScan_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_Instruction.hpp"
+#include "c1/c1_LinearScan.hpp"
+#include "utilities/bitMap.inline.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_LinearScan_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2005, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_C1_LINEARSCAN_X86_HPP
+#define CPU_X86_VM_C1_LINEARSCAN_X86_HPP
 inline bool LinearScan::is_processed_reg_num(int reg_num) {
 #ifndef _LP64
 …
   assert(FrameMap::rbp_opr->cpu_regnr() == 7, "wrong assumption below");
   assert(reg_num >= 0, "invalid reg_num");
-  return reg_num < 6 || reg_num > 7;
 #else
+  // rsp and rbp, r10, r15 (numbers 6 ancd 7) are ignored
+  // rsp and rbp, r10, r15 (numbers [12,15]) are ignored
+  // r12 (number 11) is conditional on compressed oops.
+  assert(FrameMap::r12_opr->cpu_regnr() == 11, "wrong assumption below");
   assert(FrameMap::r10_opr->cpu_regnr() == 12, "wrong assumption below");
   assert(FrameMap::r15_opr->cpu_regnr() == 13, "wrong assumption below");
 …
   assert(FrameMap::rbp_opr->cpu_regnrLo() == 15, "wrong assumption below");
   assert(reg_num >= 0, "invalid reg_num");
-  return reg_num < 12 || reg_num > 15;
 #endif // _LP64
+  return reg_num <= FrameMap::last_cpu_reg() || reg_num >= pd_nof_cpu_regs_frame_map;
+}
 …
     assert(cur->type() != T_FLOAT && cur->type() != T_DOUBLE, "cpu regs only");
     _first_reg = pd_first_byte_reg;
     _last_reg = pd_last_byte_reg;
+    _last_reg = FrameMap::last_byte_reg();
     return true;
   } else if ((UseSSE >= 1 && cur->type() == T_FLOAT) || (UseSSE >= 2 && cur->type() == T_DOUBLE)) {
 …
   void allocate();
 };
+#endif // CPU_X86_VM_C1_LINEARSCAN_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_c1_MacroAssembler_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "c1/c1_Runtime1.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "gc_interface/collectedHeap.hpp"
+#include "interpreter/interpreter.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/markOop.hpp"
+#include "runtime/basicLock.hpp"
+#include "runtime/biasedLocking.hpp"
+#include "runtime/os.hpp"
+#include "runtime/stubRoutines.hpp"
 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
 …
   } else {
     eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
+    incr_allocated_bytes(noreg, var_size_in_bytes, con_size_in_bytes, t1);
+  }
+}
 …
     movptr(Address(obj, oopDesc::mark_offset_in_bytes ()), (int32_t)(intptr_t)markOopDesc::prototype());
+  }
+  movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
+#ifdef _LP64
+  if (UseCompressedOops) { // Take care not to kill klass
+    movptr(t1, klass);
+    encode_heap_oop_not_null(t1);
+    movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
+  } else
+#endif
+  {
+    movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
+  }
   if (len->is_valid()) {
     movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
+  }
+#ifdef _LP64
+  else if (UseCompressedOops) {
+    xorptr(t1, t1);
+    store_klass_gap(obj, t1);
+  }
+#endif
+}
 …
 void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) {
   assert(obj == rax, "obj must be in rax, for cmpxchg");
   assert(obj != t1 && obj != t2 && t1 != t2, "registers must be different"); // XXX really?
+  assert_different_registers(obj, t1, t2); // XXX really?
   assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
 …
   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
          "con_size_in_bytes is not multiple of alignment");
   const int hdr_size_in_bytes = instanceOopDesc::base_offset_in_bytes();
+  const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
   initialize_header(obj, klass, noreg, t1, t2);
 …
   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
   int start_offset = offset();
+  cmpptr(iCache, Address(receiver, oopDesc::klass_offset_in_bytes()));
+  if (UseCompressedOops) {
+    load_klass(rscratch1, receiver);
+    cmpptr(rscratch1, iCache);
+  } else {
+    cmpptr(iCache, Address(receiver, oopDesc::klass_offset_in_bytes()));
+  }
   // if icache check fails, then jump to runtime routine
   // Note: RECEIVER must still contain the receiver!
 …
           RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
   assert(offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry");
+  assert(UseCompressedOops || offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry");
+}

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_MacroAssembler_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_C1_MACROASSEMBLER_X86_HPP
+#define CPU_X86_VM_C1_MACROASSEMBLER_X86_HPP
 // C1_MacroAssembler contains high-level macros for C1
 …
   void invalidate_registers(bool inv_rax, bool inv_rbx, bool inv_rcx, bool inv_rdx, bool inv_rsi, bool inv_rdi) PRODUCT_RETURN;
+#endif // CPU_X86_VM_C1_MACROASSEMBLER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_Runtime1_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_c1_Runtime1_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "c1/c1_Defs.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "c1/c1_Runtime1.hpp"
+#include "interpreter/interpreter.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/compiledICHolderOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "register_x86.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/signature.hpp"
+#include "runtime/vframeArray.hpp"
+#include "vmreg_x86.inline.hpp"
 …
         __ verify_not_null_oop(exception_oop);
         oop_maps = new OopMapSet();
         OopMap* oop_map = generate_oop_map(sasm, 1);
 …
           // refilling the TLAB or allocating directly from eden.
           Label retry_tlab, try_eden;
+          __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy rdx (klass)
+          const Register thread =
+            __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy rdx (klass), returns rdi
           __ bind(retry_tlab);
 …
           // get the instance size (size is postive so movl is fine for 64bit)
           __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
           __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
           __ verify_oop(obj);
 …
           // get the instance size (size is postive so movl is fine for 64bit)
           __ movl(obj_size, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
           __ eden_allocate(obj, obj_size, 0, t1, slow_path);
+          __ incr_allocated_bytes(thread, obj_size, 0);
           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
           __ verify_oop(obj);
 …
       break;
-#ifdef TIERED
     case counter_overflow_id:
+      {
         Register bci = rax;
+        Register bci = rax, method = rbx;
         __ enter();
         OopMap* map = save_live_registers(sasm, 2);
+        OopMap* map = save_live_registers(sasm, 3);
         // Retrieve bci
         __ movl(bci, Address(rbp, 2*BytesPerWord));
+        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci);
+        // And a pointer to the methodOop
+        __ movptr(method, Address(rbp, 3*BytesPerWord));
+        int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
         oop_maps = new OopMapSet();
         oop_maps->add_gc_map(call_offset, map);
 …
+      }
       break;
-#endif // TIERED
     case new_type_array_id:
 …
           // refilling the TLAB or allocating directly from eden.
           Label retry_tlab, try_eden;
+          __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves rbx, & rdx
+          const Register thread =
+            __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves rbx & rdx, returns rdi
           __ bind(retry_tlab);
           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
           // since size is postive movl does right thing on 64bit
+          // since size is positive movl does right thing on 64bit
           __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
           // since size is postive movl does right thing on 64bit
 …
           __ bind(try_eden);
           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
           // since size is postive movl does right thing on 64bit
+          // since size is positive movl does right thing on 64bit
           __ movl(t1, Address(klass, klassOopDesc::header_size() * HeapWordSize + Klass::layout_helper_offset_in_bytes()));
           // since size is postive movl does right thing on 64bit
 …
           __ eden_allocate(obj, arr_size, 0, t1, slow_path);  // preserves arr_size
+          __ incr_allocated_bytes(thread, arr_size, 0);
           __ initialize_header(obj, klass, length, t1, t2);
 …
         Label register_finalizer;
         Register t = rsi;
         __ movptr(t, Address(rax, oopDesc::klass_offset_in_bytes()));
+        __ load_klass(t, rax);
         __ movl(t, Address(t, Klass::access_flags_offset_in_bytes() + sizeof(oopDesc)));
         __ testl(t, JVM_ACC_HAS_FINALIZER);

trunk/openjdk/hotspot/src/cpu/x86/vm/c1_globals_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_C1_GLOBALS_X86_HPP
+#define CPU_X86_VM_C1_GLOBALS_X86_HPP
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
 // Sets the default values for platform dependent flags used by the client compiler.
 // (see c1_globals.hpp)
 …
 define_pd_global(bool, TieredCompilation,            false);
 define_pd_global(intx, CompileThreshold,             1500 );
-define_pd_global(intx, Tier2CompileThreshold,        1500 );
-define_pd_global(intx, Tier3CompileThreshold,        2500 );
-define_pd_global(intx, Tier4CompileThreshold,        4500 );
 define_pd_global(intx, BackEdgeThreshold,            100000);
-define_pd_global(intx, Tier2BackEdgeThreshold,       100000);
-define_pd_global(intx, Tier3BackEdgeThreshold,       100000);
-define_pd_global(intx, Tier4BackEdgeThreshold,       100000);
 define_pd_global(intx, OnStackReplacePercentage,     933  );
 …
 define_pd_global(intx, SafepointPollOffset,          256  );
+#endif // CPU_X86_VM_C1_GLOBALS_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c2_globals_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_C2_GLOBALS_X86_HPP
+#define CPU_X86_VM_C2_GLOBALS_X86_HPP
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
 // Sets the default values for platform dependent flags used by the server compiler.
 // (see c2_globals.hpp).  Alpha-sorted.
 …
 #endif // CC_INTERP
 define_pd_global(bool, TieredCompilation,            false);
-#ifdef TIERED
-define_pd_global(intx, CompileThreshold,             1000);
-#else
 define_pd_global(intx, CompileThreshold,             10000);
-#endif // TIERED
-define_pd_global(intx, Tier2CompileThreshold,        10000);
-define_pd_global(intx, Tier3CompileThreshold,        20000);
-define_pd_global(intx, Tier4CompileThreshold,        40000);
 define_pd_global(intx, BackEdgeThreshold,            100000);
-define_pd_global(intx, Tier2BackEdgeThreshold,       100000);
-define_pd_global(intx, Tier3BackEdgeThreshold,       100000);
-define_pd_global(intx, Tier4BackEdgeThreshold,       100000);
 define_pd_global(intx, OnStackReplacePercentage,     140);
 …
 // Ergonomics related flags
 define_pd_global(bool, NeverActAsServerClassMachine, false);
+#endif // CPU_X86_VM_C2_GLOBALS_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/c2_init_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_c2_init_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "opto/compile.hpp"
+#include "opto/node.hpp"
 // processor dependent initialization for i486

trunk/openjdk/hotspot/src/cpu/x86/vm/codeBuffer_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2002, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_CODEBUFFER_X86_HPP
+#define CPU_X86_VM_CODEBUFFER_X86_HPP
 private:
   void pd_initialize() {}
 …
 public:
   void flush_bundle(bool start_new_bundle) {}
+#endif // CPU_X86_VM_CODEBUFFER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/copy_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2004, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_COPY_X86_HPP
+#define CPU_X86_VM_COPY_X86_HPP
 // Inline functions for memory copy and fill.
 // Contains inline asm implementations
+#include "incls/_copy_pd.inline.hpp.incl"
+#ifdef TARGET_OS_ARCH_linux_x86
+# include "copy_linux_x86.inline.hpp"
+#endif
+#ifdef TARGET_OS_ARCH_solaris_x86
+# include "copy_solaris_x86.inline.hpp"
+#endif
+#ifdef TARGET_OS_ARCH_windows_x86
+# include "copy_windows_x86.inline.hpp"
+#endif
 static void pd_fill_to_words(HeapWord* tohw, size_t count, juint value) {
 …
   (void)memset(to, 0, count);
+}
+#endif // CPU_X86_VM_COPY_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/cppInterpreterGenerator_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_CPPINTERPRETERGENERATOR_X86_HPP
+#define CPU_X86_VM_CPPINTERPRETERGENERATOR_X86_HPP
  protected:
 …
                                           const Register sender_sp,
                                           bool native); // C++ interpreter only
+#endif // CPU_X86_VM_CPPINTERPRETERGENERATOR_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/cppInterpreter_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2007, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_cppInterpreter_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "interpreter/bytecodeHistogram.hpp"
+#include "interpreter/cppInterpreter.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterGenerator.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/arguments.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
+#include "runtime/timer.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
+#ifdef SHARK
+#include "shark/shark_globals.hpp"
+#endif
 #ifdef CC_INTERP

trunk/openjdk/hotspot/src/cpu/x86/vm/cppInterpreter_x86.hpp

-              r278
+              r309
  */
+#ifndef CPU_X86_VM_CPPINTERPRETER_X86_HPP
+#define CPU_X86_VM_CPPINTERPRETER_X86_HPP
   protected:
 …
   // Max size with JVMTI
   const static int InterpreterCodeSize = 168 * 1024;
+#endif // CPU_X86_VM_CPPINTERPRETER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/debug_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1999, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_debug_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "code/codeCache.hpp"
+#include "code/nmethod.hpp"
+#include "runtime/frame.hpp"
+#include "runtime/init.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/top.hpp"
 void pd_ps(frame f) {}

trunk/openjdk/hotspot/src/cpu/x86/vm/depChecker_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_depChecker_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "compiler/disassembler.hpp"
+#include "depChecker_x86.hpp"
 // Nothing to do on i486

trunk/openjdk/hotspot/src/cpu/x86/vm/depChecker_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_DEPCHECKER_X86_HPP
+#define CPU_X86_VM_DEPCHECKER_X86_HPP
 // Nothing to do on i486
+#endif // CPU_X86_VM_DEPCHECKER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/disassembler_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_DISASSEMBLER_X86_HPP
+#define CPU_X86_VM_DISASSEMBLER_X86_HPP
   static int pd_instruction_alignment() {
     return 1;
 …
     return "";
+  }
+#endif // CPU_X86_VM_DISASSEMBLER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/dump_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2004, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_dump_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "memory/compactingPermGenGen.hpp"
+#include "memory/generation.inline.hpp"
+#include "memory/space.inline.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/dump_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2004, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_dump_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "memory/compactingPermGenGen.hpp"
+#include "memory/generation.inline.hpp"
+#include "memory/space.inline.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/frame_x86.cpp

-              r278
+              r309
  */
+# include "incls/_precompiled.incl"
+# include "incls/_frame_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/interpreter.hpp"
+#include "memory/resourceArea.hpp"
+#include "oops/markOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/javaCalls.hpp"
+#include "runtime/monitorChunk.hpp"
+#include "runtime/signature.hpp"
+#include "runtime/stubCodeGenerator.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Runtime1.hpp"
+#include "runtime/vframeArray.hpp"
+#endif
 #ifdef ASSERT
 …
     // Could just be some random pointer within the codeBlob
+    if (!sender_blob->instructions_contains(sender_pc)) return false;
+    if (!sender_blob->code_contains(sender_pc)) {
+      return false;
+    }
     // We should never be able to see an adapter if the current frame is something from code cache
+    if ( sender_blob->is_adapter_blob()) {
+    if (sender_blob->is_adapter_blob()) {
       return false;
+    }
 …
   address original_pc = nm->get_original_pc(&fr);
   assert(nm->code_contains(original_pc), "original PC must be in nmethod");
+  assert(nm->insts_contains(original_pc), "original PC must be in nmethod");
   assert(nm->is_method_handle_return(original_pc) == is_method_handle_return, "must be");
+}

trunk/openjdk/hotspot/src/cpu/x86/vm/frame_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_FRAME_X86_HPP
+#define CPU_X86_VM_FRAME_X86_HPP
+#include "runtime/synchronizer.hpp"
+#include "utilities/top.hpp"
 // A frame represents a physical stack frame (an activation).  Frames can be
 …
   inline interpreterState get_interpreterState() const;
 #endif // CC_INTERP
+#endif // CPU_X86_VM_FRAME_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/frame_x86.inline.hpp

-              r278
+              r309
+ *
  */
+#ifndef CPU_X86_VM_FRAME_X86_INLINE_HPP
+#define CPU_X86_VM_FRAME_X86_INLINE_HPP
 // Inline functions for Intel frames:
 …
   if (original_pc != NULL) {
     _pc = original_pc;
     assert(((nmethod*)_cb)->code_contains(_pc), "original PC must be in nmethod");
+    assert(((nmethod*)_cb)->insts_contains(_pc), "original PC must be in nmethod");
     _deopt_state = is_deoptimized;
   } else {
 …
   *((oop*) map->location(rax->as_VMReg())) = obj;
+}
+#endif // CPU_X86_VM_FRAME_X86_INLINE_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/globalDefinitions_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1999, 2004, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_GLOBALDEFINITIONS_X86_HPP
+#define CPU_X86_VM_GLOBALDEFINITIONS_X86_HPP
 const int StackAlignmentInBytes  = 16;
+#endif // CPU_X86_VM_GLOBALDEFINITIONS_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/globals_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_GLOBALS_X86_HPP
+#define CPU_X86_VM_GLOBALS_X86_HPP
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
 // Sets the default values for platform dependent flags used by the runtime system.
 …
 define_pd_global(intx, StackShadowPages, SOLARIS_ONLY(20) NOT_SOLARIS(6) DEBUG_ONLY(+2));
 #else
 define_pd_global(intx, StackShadowPages, 3 DEBUG_ONLY(+1));
+define_pd_global(intx, StackShadowPages, 3 DEBUG_ONLY(+5));
 #endif // AMD64
 …
 define_pd_global(bool, RewriteBytecodes,     true);
 define_pd_global(bool, RewriteFrequentPairs, true);
+define_pd_global(bool, UseMembar,            false);
+#endif // CPU_X86_VM_GLOBALS_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/icBuffer_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2006, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_icBuffer_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "code/icBuffer.hpp"
+#include "gc_interface/collectedHeap.inline.hpp"
+#include "interpreter/bytecodes.hpp"
+#include "memory/resourceArea.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/oop.inline.hpp"
+#include "oops/oop.inline2.hpp"
 int InlineCacheBuffer::ic_stub_code_size() {

trunk/openjdk/hotspot/src/cpu/x86/vm/icache_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_icache_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "runtime/icache.hpp"
 #define __ _masm->

trunk/openjdk/hotspot/src/cpu/x86/vm/icache_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2004, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_ICACHE_X86_HPP
+#define CPU_X86_VM_ICACHE_X86_HPP
 // Interface for updating the instruction cache.  Whenever the VM modifies
 …
 #endif // AMD64
 };
+#endif // CPU_X86_VM_ICACHE_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/interp_masm_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interp_masm_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "interp_masm_x86_32.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/markOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiRedefineClassesTrace.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/basicLock.hpp"
+#include "runtime/biasedLocking.hpp"
+#include "runtime/sharedRuntime.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "thread_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "thread_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "thread_windows.inline.hpp"
+#endif
 …
 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
   assert(ProfileInterpreter, "must be profiling interpreter");
   Label zero_continue;
+  Label set_mdp;
   push(rax);
   push(rbx);
 …
   movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
   testptr(rax, rax);
+  jcc(Assembler::zero, zero_continue);
+  jcc(Assembler::zero, set_mdp);
   // rbx,: method
   // rsi: bcp
   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, rsi);
   // rax,: mdi
+  // mdo is guaranteed to be non-zero here, we checked for it before the call.
   movptr(rbx, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
-  testptr(rbx, rbx);
-  jcc(Assembler::zero, zero_continue);
   addptr(rbx, in_bytes(methodDataOopDesc::data_offset()));
+  addptr(rbx, rax);
+  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rbx);
+  bind(zero_continue);
+  addptr(rax, rbx);
+  bind(set_mdp);
+  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax);
   pop(rbx);
   pop(rax);
 …
+  }
+}
+// Jump if ((*counter_addr += increment) & mask) satisfies the condition.
+void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
+                                                        int increment, int mask,
+                                                        Register scratch, bool preloaded,
+                                                        Condition cond, Label* where) {
+  if (!preloaded) {
+    movl(scratch, counter_addr);
+  }
+  incrementl(scratch, increment);
+  movl(counter_addr, scratch);
+  andl(scratch, mask);
+  jcc(cond, *where);
+}

trunk/openjdk/hotspot/src/cpu/x86/vm/interp_masm_x86_32.hpp

-              r278
+              r309
+ *
  */
+#ifndef CPU_X86_VM_INTERP_MASM_X86_32_HPP
+#define CPU_X86_VM_INTERP_MASM_X86_32_HPP
+#include "assembler_x86.inline.hpp"
+#include "interpreter/invocationCounter.hpp"
 // This file specializes the assember with interpreter-specific macros
 …
   void increment_mdp_data_at(Register mdp_in, Register reg, int constant,
                              bool decrement = false);
+  void increment_mask_and_jump(Address counter_addr,
+                               int increment, int mask,
+                               Register scratch, bool preloaded,
+                               Condition cond, Label* where);
   void set_mdp_flag_at(Register mdp_in, int flag_constant);
   void test_mdp_data_at(Register mdp_in, int offset, Register value,
 …
 };
+#endif // CPU_X86_VM_INTERP_MASM_X86_32_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/interp_masm_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interp_masm_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "interp_masm_x86_64.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/markOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiRedefineClassesTrace.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/basicLock.hpp"
+#include "runtime/biasedLocking.hpp"
+#include "runtime/sharedRuntime.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "thread_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "thread_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "thread_windows.inline.hpp"
+#endif
 …
     // compiled code in threads for which the event is enabled.  Check here for
     // interp_only_mode if these events CAN be enabled.
-    get_thread(temp);
     // interp_only is an int, on little endian it is sufficient to test the byte only
     // Is a cmpl faster (ce
     cmpb(Address(temp, JavaThread::interp_only_mode_offset()), 0);
+    // Is a cmpl faster?
+    cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0);
     jcc(Assembler::zero, run_compiled_code);
     jmp(Address(method, methodOopDesc::interpreter_entry_offset()));
 …
 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
   assert(ProfileInterpreter, "must be profiling interpreter");
   Label zero_continue;
+  Label set_mdp;
   push(rax);
   push(rbx);
 …
   movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
   testptr(rax, rax);
+  jcc(Assembler::zero, zero_continue);
+  jcc(Assembler::zero, set_mdp);
   // rbx: method
   // r13: bcp
   call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), rbx, r13);
   // rax: mdi
+  // mdo is guaranteed to be non-zero here, we checked for it before the call.
   movptr(rbx, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
-  testptr(rbx, rbx);
-  jcc(Assembler::zero, zero_continue);
   addptr(rbx, in_bytes(methodDataOopDesc::data_offset()));
+  addptr(rbx, rax);
+  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rbx);
+  bind(zero_continue);
+  addptr(rax, rbx);
+  bind(set_mdp);
+  movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax);
   pop(rbx);
   pop(rax);
 …
+  }
+}
+// Jump if ((*counter_addr += increment) & mask) satisfies the condition.
+void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
+                                                        int increment, int mask,
+                                                        Register scratch, bool preloaded,
+                                                        Condition cond, Label* where) {
+  if (!preloaded) {
+    movl(scratch, counter_addr);
+  }
+  incrementl(scratch, increment);
+  movl(counter_addr, scratch);
+  andl(scratch, mask);
+  jcc(cond, *where);
+}

trunk/openjdk/hotspot/src/cpu/x86/vm/interp_masm_x86_64.hpp

-              r278
+              r309
+ *
  */
+#ifndef CPU_X86_VM_INTERP_MASM_X86_64_HPP
+#define CPU_X86_VM_INTERP_MASM_X86_64_HPP
+#include "assembler_x86.inline.hpp"
+#include "interpreter/invocationCounter.hpp"
 // This file specializes the assember with interpreter-specific macros
 …
   void increment_mdp_data_at(Register mdp_in, Register reg, int constant,
                              bool decrement = false);
+  void increment_mask_and_jump(Address counter_addr,
+                               int increment, int mask,
+                               Register scratch, bool preloaded,
+                               Condition cond, Label* where);
   void set_mdp_flag_at(Register mdp_in, int flag_constant);
   void test_mdp_data_at(Register mdp_in, int offset, Register value,
 …
   void notify_method_exit(TosState state, NotifyMethodExitMode mode);
 };
+#endif // CPU_X86_VM_INTERP_MASM_X86_64_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreterGenerator_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_INTERPRETERGENERATOR_X86_HPP
+#define CPU_X86_VM_INTERPRETERGENERATOR_X86_HPP
 // Generation of Interpreter
 …
   void generate_counter_incr(Label* overflow, Label* profile_method, Label* profile_method_continue);
   void generate_counter_overflow(Label* do_continue);
+#endif // CPU_X86_VM_INTERPRETERGENERATOR_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreterRT_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_INTERPRETERRT_X86_HPP
+#define CPU_X86_VM_INTERPRETERRT_X86_HPP
+#include "memory/allocation.hpp"
 // native method calls
 …
   static Register temp();
 };
+#endif // CPU_X86_VM_INTERPRETERRT_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreterRT_x86_32.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interpreterRT_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "memory/allocation.inline.hpp"
+#include "memory/universe.inline.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/icache.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "runtime/signature.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreterRT_x86_64.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interpreterRT_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "memory/allocation.inline.hpp"
+#include "memory/universe.inline.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/icache.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "runtime/signature.hpp"
 #define __ _masm->

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreter_x86.hpp

-              r278
+              r309
  */
+#ifndef CPU_X86_VM_INTERPRETER_X86_HPP
+#define CPU_X86_VM_INTERPRETER_X86_HPP
  public:
 …
     return stackElementWords * i;
+  }
+#endif // CPU_X86_VM_INTERPRETER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreter_x86_32.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interpreter_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "interpreter/bytecodeHistogram.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterGenerator.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "interpreter/templateTable.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/arguments.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
+#include "runtime/timer.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Runtime1.hpp"
+#endif
 #define __ _masm->

trunk/openjdk/hotspot/src/cpu/x86/vm/interpreter_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interpreter_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "interpreter/bytecodeHistogram.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterGenerator.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "interpreter/templateTable.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/arguments.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
+#include "runtime/timer.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Runtime1.hpp"
+#endif
 #define __ _masm->

trunk/openjdk/hotspot/src/cpu/x86/vm/javaFrameAnchor_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2002, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_JAVAFRAMEANCHOR_X86_HPP
+#define CPU_X86_VM_JAVAFRAMEANCHOR_X86_HPP
 private:
 …
   // Assert (last_Java_sp == NULL || fp == NULL)
   void set_last_Java_fp(intptr_t* fp)                { _last_Java_fp = fp; }
+#endif // CPU_X86_VM_JAVAFRAMEANCHOR_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/jniFastGetField_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2004, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_jniFastGetField_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "prims/jniFastGetField.hpp"
+#include "prims/jvm_misc.hpp"
+#include "runtime/safepoint.hpp"
 #define __ masm->
 …
+  }
   ResourceMark rm;
+  BufferBlob* b = BufferBlob::create(name, BUFFER_SIZE*wordSize);
+  address fast_entry = b->instructions_begin();
+  CodeBuffer cbuf(fast_entry, b->instructions_size());
+  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE*wordSize);
+  CodeBuffer cbuf(blob);
   MacroAssembler* masm = new MacroAssembler(&cbuf);
+  address fast_entry = __ pc();
   Label slow;
 …
 #else
   switch (type) {
     case T_BOOLEAN: jni_fast_GetBooleanField_fp = (GetBooleanField_t)fast_entry; break;
     case T_BYTE:    jni_fast_GetByteField_fp = (GetByteField_t)fast_entry; break;
     case T_CHAR:    jni_fast_GetCharField_fp = (GetCharField_t)fast_entry; break;
     case T_SHORT:   jni_fast_GetShortField_fp = (GetShortField_t)fast_entry; break;
     case T_INT:     jni_fast_GetIntField_fp = (GetIntField_t)fast_entry;
+  case T_BOOLEAN: jni_fast_GetBooleanField_fp = (GetBooleanField_t) fast_entry; break;
+  case T_BYTE:    jni_fast_GetByteField_fp    = (GetByteField_t)    fast_entry; break;
+  case T_CHAR:    jni_fast_GetCharField_fp    = (GetCharField_t)    fast_entry; break;
+  case T_SHORT:   jni_fast_GetShortField_fp   = (GetShortField_t)   fast_entry; break;
+  case T_INT:     jni_fast_GetIntField_fp     = (GetIntField_t)     fast_entry; break;
+  }
   return os::win32::fast_jni_accessor_wrapper(type);
 …
   const char *name = "jni_fast_GetLongField";
   ResourceMark rm;
+  BufferBlob* b = BufferBlob::create(name, BUFFER_SIZE*wordSize);
+  address fast_entry = b->instructions_begin();
+  CodeBuffer cbuf(fast_entry, b->instructions_size());
+  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE*wordSize);
+  CodeBuffer cbuf(blob);
   MacroAssembler* masm = new MacroAssembler(&cbuf);
+  address fast_entry = __ pc();
   Label slow;
 …
   return fast_entry;
 #else
   jni_fast_GetLongField_fp = (GetLongField_t)fast_entry;
+  jni_fast_GetLongField_fp = (GetLongField_t) fast_entry;
   return os::win32::fast_jni_accessor_wrapper(T_LONG);
 #endif
 …
+  }
   ResourceMark rm;
+  BufferBlob* b = BufferBlob::create(name, BUFFER_SIZE*wordSize);
+  address fast_entry = b->instructions_begin();
+  CodeBuffer cbuf(fast_entry, b->instructions_size());
+  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE*wordSize);
+  CodeBuffer cbuf(blob);
   MacroAssembler* masm = new MacroAssembler(&cbuf);
+  address fast_entry = __ pc();
   Label slow_with_pop, slow;
 …
 #else
   switch (type) {
     case T_FLOAT:  jni_fast_GetFloatField_fp = (GetFloatField_t)fast_entry; break;
     case T_DOUBLE: jni_fast_GetDoubleField_fp = (GetDoubleField_t)fast_entry;
+  case T_FLOAT:  jni_fast_GetFloatField_fp  = (GetFloatField_t)  fast_entry; break;
+  case T_DOUBLE: jni_fast_GetDoubleField_fp = (GetDoubleField_t) fast_entry; break;
+  }
   return os::win32::fast_jni_accessor_wrapper(type);

trunk/openjdk/hotspot/src/cpu/x86/vm/jniFastGetField_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2004, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_jniFastGetField_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "prims/jniFastGetField.hpp"
+#include "prims/jvm_misc.hpp"
+#include "runtime/safepoint.hpp"
 #define __ masm->
 …
+  }
   ResourceMark rm;
+  BufferBlob* b = BufferBlob::create(name, BUFFER_SIZE);
+  address fast_entry = b->instructions_begin();
+  CodeBuffer cbuf(fast_entry, b->instructions_size());
+  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE);
+  CodeBuffer cbuf(blob);
   MacroAssembler* masm = new MacroAssembler(&cbuf);
+  address fast_entry = __ pc();
   Label slow;
 …
+  }
   ResourceMark rm;
+  BufferBlob* b = BufferBlob::create(name, BUFFER_SIZE);
+  address fast_entry = b->instructions_begin();
+  CodeBuffer cbuf(fast_entry, b->instructions_size());
+  BufferBlob* blob = BufferBlob::create(name, BUFFER_SIZE);
+  CodeBuffer cbuf(blob);
   MacroAssembler* masm = new MacroAssembler(&cbuf);
+  address fast_entry = __ pc();
   Label slow;

trunk/openjdk/hotspot/src/cpu/x86/vm/jniTypes_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, 2003, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_JNITYPES_X86_HPP
+#define CPU_X86_VM_JNITYPES_X86_HPP
+#include "memory/allocation.hpp"
+#include "oops/oop.hpp"
+#include "prims/jni.h"
 // This file holds platform-dependent routines used to write primitive jni
 …
 #undef _JNI_SLOT_OFFSET
 };
+#endif // CPU_X86_VM_JNITYPES_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/jni_x86.h

r278	r309
1	1	/*
2		* Copyright (c) 1997, 2009, Oracle and/or its affiliates. All rights reserved.
	2	* Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
3	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4	4	*

trunk/openjdk/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_methodHandles_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/interpreter.hpp"
+#include "memory/allocation.inline.hpp"
+#include "prims/methodHandles.hpp"
 #define __ _masm->
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
 address MethodHandleEntry::start_compiled_entry(MacroAssembler* _masm,
 …
   // Verify that argslot lies within (rsp, rbp].
   Label L_ok, L_bad;
+  BLOCK_COMMENT("{ verify_argslot");
   __ cmpptr(argslot_reg, rbp);
   __ jccb(Assembler::above, L_bad);
 …
   __ stop(error_message);
   __ bind(L_ok);
+  BLOCK_COMMENT("} verify_argslot");
+}
 #endif
 …
   // rcx: receiver method handle (must load from sp[MethodTypeForm.vmslots])
   // rsi/r13: sender SP (must preserve; see prepare_to_jump_from_interpreted)
   // rdx: garbage temp, blown away
+  // rdx, rdi: garbage temp, blown away
   Register rbx_method = rbx;
 …
   Register rax_mtype  = rax;
   Register rdx_temp   = rdx;
+  Register rdi_temp   = rdi;
   // emit WrongMethodType path first, to enable jccb back-branch from main path
   Label wrong_method_type;
   __ bind(wrong_method_type);
+  Label invoke_generic_slow_path;
+  assert(methodOopDesc::intrinsic_id_size_in_bytes() == sizeof(u1), "");;
+  __ cmpb(Address(rbx_method, methodOopDesc::intrinsic_id_offset_in_bytes()), (int) vmIntrinsics::_invokeExact);
+  __ jcc(Assembler::notEqual, invoke_generic_slow_path);
   __ push(rax_mtype);       // required mtype
   __ push(rcx_recv);        // bad mh (1st stacked argument)
 …
+    }
+  }
-  Register rbx_temp = rbx_method; // done with incoming methodOop
   // given the MethodType, find out where the MH argument is buried
+  __ movptr(rdx_temp, Address(rax_mtype,
+                              __ delayed_value(java_dyn_MethodType::form_offset_in_bytes, rbx_temp)));
+  __ movl(rdx_temp, Address(rdx_temp,
+                            __ delayed_value(java_dyn_MethodTypeForm::vmslots_offset_in_bytes, rbx_temp)));
+  __ movptr(rcx_recv, __ argument_address(rdx_temp));
+  __ check_method_handle_type(rax_mtype, rcx_recv, rdx_temp, wrong_method_type);
+  __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+  __ load_heap_oop(rdx_temp, Address(rax_mtype, __ delayed_value(java_dyn_MethodType::form_offset_in_bytes, rdi_temp)));
+  Register rdx_vmslots = rdx_temp;
+  __ movl(rdx_vmslots, Address(rdx_temp, __ delayed_value(java_dyn_MethodTypeForm::vmslots_offset_in_bytes, rdi_temp)));
+  __ movptr(rcx_recv, __ argument_address(rdx_vmslots));
+  trace_method_handle(_masm, "invokeExact");
+  __ check_method_handle_type(rax_mtype, rcx_recv, rdi_temp, wrong_method_type);
+  __ jump_to_method_handle_entry(rcx_recv, rdi_temp);
+  // for invokeGeneric (only), apply argument and result conversions on the fly
+  __ bind(invoke_generic_slow_path);
+#ifdef ASSERT
+  { Label L;
+    __ cmpb(Address(rbx_method, methodOopDesc::intrinsic_id_offset_in_bytes()), (int) vmIntrinsics::_invokeGeneric);
+    __ jcc(Assembler::equal, L);
+    __ stop("bad methodOop::intrinsic_id");
+    __ bind(L);
+  }
+#endif //ASSERT
+  Register rbx_temp = rbx_method;  // don't need it now
+  // make room on the stack for another pointer:
+  Register rcx_argslot = rcx_recv;
+  __ lea(rcx_argslot, __ argument_address(rdx_vmslots, 1));
+  insert_arg_slots(_masm, 2 * stack_move_unit(), _INSERT_REF_MASK,
+                   rcx_argslot, rbx_temp, rdx_temp);
+  // load up an adapter from the calling type (Java weaves this)
+  __ load_heap_oop(rdx_temp, Address(rax_mtype, __ delayed_value(java_dyn_MethodType::form_offset_in_bytes, rdi_temp)));
+  Register rdx_adapter = rdx_temp;
+  // __ load_heap_oop(rdx_adapter, Address(rdx_temp, java_dyn_MethodTypeForm::genericInvoker_offset_in_bytes()));
+  // deal with old JDK versions:
+  __ lea(rdi_temp, Address(rdx_temp, __ delayed_value(java_dyn_MethodTypeForm::genericInvoker_offset_in_bytes, rdi_temp)));
+  __ cmpptr(rdi_temp, rdx_temp);
+  Label sorry_no_invoke_generic;
+  __ jcc(Assembler::below, sorry_no_invoke_generic);
+  __ load_heap_oop(rdx_adapter, Address(rdi_temp, 0));
+  __ testptr(rdx_adapter, rdx_adapter);
+  __ jcc(Assembler::zero, sorry_no_invoke_generic);
+  __ movptr(Address(rcx_argslot, 1 * Interpreter::stackElementSize), rdx_adapter);
+  // As a trusted first argument, pass the type being called, so the adapter knows
+  // the actual types of the arguments and return values.
+  // (Generic invokers are shared among form-families of method-type.)
+  __ movptr(Address(rcx_argslot, 0 * Interpreter::stackElementSize), rax_mtype);
+  // FIXME: assert that rdx_adapter is of the right method-type.
+  __ mov(rcx, rdx_adapter);
+  trace_method_handle(_masm, "invokeGeneric");
+  __ jump_to_method_handle_entry(rcx, rdi_temp);
+  __ bind(sorry_no_invoke_generic); // no invokeGeneric implementation available!
+  __ movptr(rcx_recv, Address(rcx_argslot, -1 * Interpreter::stackElementSize));  // recover original MH
+  __ push(rax_mtype);       // required mtype
+  __ push(rcx_recv);        // bad mh (1st stacked argument)
+  __ jump(ExternalAddress(Interpreter::throw_WrongMethodType_entry()));
   return entry_point;
 …
   //     rdx[-size] = rdx[0]
   //   argslot -= size;
+  BLOCK_COMMENT("insert_arg_slots {");
   __ mov(rdx_temp, rsp);                        // source pointer for copy
   __ lea(rsp, Address(rsp, arg_slots, Address::times_ptr));
+  {
     Label loop;
     __ bind(loop);
+    __ BIND(loop);
     // pull one word down each time through the loop
     __ movptr(rbx_temp, Address(rdx_temp, 0));
 …
   // Now move the argslot down, to point to the opened-up space.
   __ lea(rax_argslot, Address(rax_argslot, arg_slots, Address::times_ptr));
+  BLOCK_COMMENT("} insert_arg_slots");
+}
 …
 #endif
+  BLOCK_COMMENT("remove_arg_slots {");
   // Pull up everything shallower than rax_argslot.
   // Then remove the excess space on the stack.
 …
+  {
     Label loop;
     __ bind(loop);
+    __ BIND(loop);
     // pull one word up each time through the loop
     __ movptr(rbx_temp, Address(rdx_temp, 0));
 …
   // And adjust the argslot address to point at the deletion point.
   __ lea(rax_argslot, Address(rax_argslot, arg_slots, Address::times_ptr));
+  BLOCK_COMMENT("} remove_arg_slots");
+}
 …
 void trace_method_handle_stub(const char* adaptername,
                               oop mh,
+                              intptr_t* saved_regs,
                               intptr_t* entry_sp,
                               intptr_t* saved_sp,
 …
   printf("MH %s mh="INTPTR_FORMAT" sp=("INTPTR_FORMAT"+"INTX_FORMAT") stack_size="INTX_FORMAT" bp="INTPTR_FORMAT"\n",
          adaptername, (intptr_t)mh, (intptr_t)entry_sp, (intptr_t)(saved_sp - entry_sp), (intptr_t)(base_sp - last_sp), (intptr_t)saved_bp);
   if (last_sp != saved_sp)
+  if (last_sp != saved_sp && last_sp != NULL)
     printf("*** last_sp="INTPTR_FORMAT"\n", (intptr_t)last_sp);
+  if (Verbose)  print_method_handle(mh);
+  if (Verbose) {
+    printf(" reg dump: ");
+    int saved_regs_count = (entry_sp-1) - saved_regs;
+    // 32 bit: rdi rsi rbp rsp; rbx rdx rcx (*) rax
+    int i;
+    for (i = 0; i <= saved_regs_count; i++) {
+      if (i > 0 && i % 4 == 0 && i != saved_regs_count)
+        printf("\n   + dump: ");
+      printf(" %d: "INTPTR_FORMAT, i, saved_regs[i]);
+    }
+    printf("\n");
+    int stack_dump_count = 16;
+    if (stack_dump_count < (int)(saved_bp + 2 - saved_sp))
+      stack_dump_count = (int)(saved_bp + 2 - saved_sp);
+    if (stack_dump_count > 64)  stack_dump_count = 48;
+    for (i = 0; i < stack_dump_count; i += 4) {
+      printf(" dump at SP[%d] "INTPTR_FORMAT": "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT"\n",
+             i, (intptr_t) &entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
+    }
+    print_method_handle(mh);
+  }
+}
+void MethodHandles::trace_method_handle(MacroAssembler* _masm, const char* adaptername) {
+  if (!TraceMethodHandles)  return;
+  BLOCK_COMMENT("trace_method_handle {");
+  __ push(rax);
+  __ lea(rax, Address(rsp, wordSize*6)); // entry_sp
+  __ pusha();
+  // arguments:
+  __ push(rbp);               // interpreter frame pointer
+  __ push(rsi);               // saved_sp
+  __ push(rax);               // entry_sp
+  __ push(rcx);               // mh
+  __ push(rcx);
+  __ movptr(Address(rsp, 0), (intptr_t) adaptername);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, trace_method_handle_stub), 5);
+  __ popa();
+  __ pop(rax);
+  BLOCK_COMMENT("} trace_method_handle");
+}
 #endif //PRODUCT
 …
+}
+//------------------------------------------------------------------------------
+// MethodHandles::generate_method_handle_stub
+//
 // Generate an "entry" field for a method handle.
 // This determines how the method handle will respond to calls.
 …
   // - rdx: garbage temp, can blow away
   Register rcx_recv    = rcx;
   Register rax_argslot = rax;
   Register rbx_temp    = rbx;
   Register rdx_temp    = rdx;
+  const Register rcx_recv    = rcx;
+  const Register rax_argslot = rax;
+  const Register rbx_temp    = rbx;
+  const Register rdx_temp    = rdx;
   // This guy is set up by prepare_to_jump_from_interpreted (from interpreted calls)
   // and gen_c2i_adapter (from compiled calls):
+  Register saved_last_sp = LP64_ONLY(r13) NOT_LP64(rsi);
+  const Register saved_last_sp = LP64_ONLY(r13) NOT_LP64(rsi);
+  // Argument registers for _raise_exception.
+  // 32-bit: Pass first two oop/int args in registers ECX and EDX.
+  const Register rarg0_code     = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
+  const Register rarg1_actual   = LP64_ONLY(j_rarg1) NOT_LP64(rdx);
+  const Register rarg2_required = LP64_ONLY(j_rarg2) NOT_LP64(rdi);
+  assert_different_registers(rarg0_code, rarg1_actual, rarg2_required, saved_last_sp);
   guarantee(java_dyn_MethodHandle::vmentry_offset_in_bytes() != 0, "must have offsets");
 …
   address interp_entry = __ pc();
+  if (UseCompressedOops)  __ unimplemented("UseCompressedOops");
+#ifndef PRODUCT
+  if (TraceMethodHandles) {
+    __ push(rax); __ push(rbx); __ push(rcx); __ push(rdx); __ push(rsi); __ push(rdi);
+    __ lea(rax, Address(rsp, wordSize*6)); // entry_sp
+    // arguments:
+    __ push(rbp);               // interpreter frame pointer
+    __ push(rsi);               // saved_sp
+    __ push(rax);               // entry_sp
+    __ push(rcx);               // mh
+    __ push(rcx);
+    __ movptr(Address(rsp, 0), (intptr_t)entry_name(ek));
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, trace_method_handle_stub), 5);
+    __ pop(rdi); __ pop(rsi); __ pop(rdx); __ pop(rcx); __ pop(rbx); __ pop(rax);
+  }
+#endif //PRODUCT
+  trace_method_handle(_masm, entry_name(ek));
+  BLOCK_COMMENT(entry_name(ek));
   switch ((int) ek) {
   case _raise_exception:
+    {
+      // Not a real MH entry, but rather shared code for raising an exception.
+      // Extra local arguments are pushed on stack, as required type at TOS+8,
+      // failing object (or NULL) at TOS+4, failing bytecode type at TOS.
+      // Beyond those local arguments are the PC, of course.
+      Register rdx_code = rdx_temp;
+      Register rcx_fail = rcx_recv;
+      Register rax_want = rax_argslot;
+      Register rdi_pc   = rdi;
+      __ pop(rdx_code);  // TOS+0
+      __ pop(rcx_fail);  // TOS+4
+      __ pop(rax_want);  // TOS+8
+      __ pop(rdi_pc);    // caller PC
+      __ mov(rsp, rsi);   // cut the stack back to where the caller started
+      // Repush the arguments as if coming from the interpreter.
+      __ push(rdx_code);
+      __ push(rcx_fail);
+      __ push(rax_want);
+      // Not a real MH entry, but rather shared code for raising an
+      // exception.  Since we use a C2I adapter to set up the
+      // interpreter state, arguments are expected in compiler
+      // argument registers.
+      assert(raise_exception_method(), "must be set");
+      address c2i_entry = raise_exception_method()->get_c2i_entry();
+      assert(c2i_entry, "method must be linked");
+      const Register rdi_pc = rax;
+      __ pop(rdi_pc);  // caller PC
+      __ mov(rsp, saved_last_sp);  // cut the stack back to where the caller started
       Register rbx_method = rbx_temp;
       Label no_method;
+      Label L_no_method;
       // FIXME: fill in _raise_exception_method with a suitable sun.dyn method
       __ movptr(rbx_method, ExternalAddress((address) &_raise_exception_method));
       __ testptr(rbx_method, rbx_method);
+      __ jccb(Assembler::zero, no_method);
+      int jobject_oop_offset = 0;
+      __ jccb(Assembler::zero, L_no_method);
+      const int jobject_oop_offset = 0;
       __ movptr(rbx_method, Address(rbx_method, jobject_oop_offset));  // dereference the jobject
       __ testptr(rbx_method, rbx_method);
       __ jccb(Assembler::zero, no_method);
+      __ jccb(Assembler::zero, L_no_method);
       __ verify_oop(rbx_method);
+      __ push(rdi_pc);          // and restore caller PC
+      __ jmp(rbx_method_fie);
+      // 32-bit: push remaining arguments as if coming from the compiler.
+      NOT_LP64(__ push(rarg2_required));
+      __ push(rdi_pc);  // restore caller PC
+      __ jump(ExternalAddress(c2i_entry));  // do C2I transition
       // If we get here, the Java runtime did not do its job of creating the exception.
       // Do something that is at least causes a valid throw from the interpreter.
+      __ bind(no_method);
+      __ pop(rax_want);
+      __ pop(rcx_fail);
+      __ push(rax_want);
+      __ push(rcx_fail);
+      __ bind(L_no_method);
+      __ push(rarg2_required);
+      __ push(rarg1_actual);
       __ jump(ExternalAddress(Interpreter::throw_WrongMethodType_entry()));
+    }
 …
+    {
       Register rbx_method = rbx_temp;
       __ movptr(rbx_method, rcx_mh_vmtarget); // target is a methodOop
+      __ load_heap_oop(rbx_method, rcx_mh_vmtarget); // target is a methodOop
       __ verify_oop(rbx_method);
       // same as TemplateTable::invokestatic or invokespecial,
 …
       Register rdx_intf  = rdx_temp;
       Register rbx_index = rbx_temp;
       __ movptr(rdx_intf,  rcx_mh_vmtarget);
       __ movl(rbx_index,   rcx_dmh_vmindex);
+      __ load_heap_oop(rdx_intf, rcx_mh_vmtarget);
+      __ movl(rbx_index, rcx_dmh_vmindex);
       __ movptr(rcx_recv, __ argument_address(rax_argslot, -1));
       __ null_check(rcx_recv, oopDesc::klass_offset_in_bytes());
 …
       // Throw an exception.
       // For historical reasons, it will be IncompatibleClassChangeError.
+      __ pushptr(Address(rdx_intf, java_mirror_offset));  // required interface
+      __ push(rcx_recv);        // bad receiver
+      __ push((int)Bytecodes::_invokeinterface);  // who is complaining?
+      __ mov(rbx_temp, rcx_recv);  // rarg2_required might be RCX
+      assert_different_registers(rarg2_required, rbx_temp);
+      __ movptr(rarg2_required, Address(rdx_intf, java_mirror_offset));  // required interface
+      __ mov(   rarg1_actual,   rbx_temp);                               // bad receiver
+      __ movl(  rarg0_code,     (int) Bytecodes::_invokeinterface);      // who is complaining?
       __ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
+    }
 …
       // store bound argument into the new stack slot:
       __ movptr(rbx_temp, rcx_bmh_argument);
+      __ load_heap_oop(rbx_temp, rcx_bmh_argument);
       Address prim_value_addr(rbx_temp, java_lang_boxing_object::value_offset_in_bytes(arg_type));
       if (arg_type == T_OBJECT) {
 …
       if (direct_to_method) {
         Register rbx_method = rbx_temp;
         __ movptr(rbx_method, rcx_mh_vmtarget);
+        __ load_heap_oop(rbx_method, rcx_mh_vmtarget);
         __ verify_oop(rbx_method);
         __ jmp(rbx_method_fie);
       } else {
         __ movptr(rcx_recv, rcx_mh_vmtarget);
+        __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
         __ verify_oop(rcx_recv);
         __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
 …
   case _adapter_retype_raw:
     // immediately jump to the next MH layer:
     __ movptr(rcx_recv, rcx_mh_vmtarget);
+    __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
     __ verify_oop(rcx_recv);
     __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
 …
       // What class are we casting to?
       __ movptr(rbx_klass, rcx_amh_argument); // this is a Class object!
       __ movptr(rbx_klass, Address(rbx_klass, java_lang_Class::klass_offset_in_bytes()));
+      __ load_heap_oop(rbx_klass, rcx_amh_argument); // this is a Class object!
+      __ load_heap_oop(rbx_klass, Address(rbx_klass, java_lang_Class::klass_offset_in_bytes()));
       Label done;
       __ movptr(rdx_temp, vmarg);
       __ testptr(rdx_temp, rdx_temp);
       __ jccb(Assembler::zero, done);         // no cast if null
+      __ jcc(Assembler::zero, done);         // no cast if null
       __ load_klass(rdx_temp, rdx_temp);
 …
       __ movptr(rdx_temp, vmarg);
+      __ pushptr(rcx_amh_argument); // required class
+      __ push(rdx_temp);            // bad object
+      __ push((int)Bytecodes::_checkcast);  // who is complaining?
+      assert_different_registers(rarg2_required, rdx_temp);
+      __ load_heap_oop(rarg2_required, rcx_amh_argument);             // required class
+      __ mov(          rarg1_actual,   rdx_temp);                     // bad object
+      __ movl(         rarg0_code,     (int) Bytecodes::_checkcast);  // who is complaining?
       __ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
       __ bind(done);
       // get the new MH:
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+    }
 …
       // get the new MH:
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       // (now we are done with the old MH)
 …
+      }
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+    }
 …
+      }
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+    }
 …
+      }
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+    }
 …
       __ pop(rdi);              // restore temp
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+    }
 …
       __ pop(rdi);              // restore temp
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
+    }
 …
       // Check the array type.
       Register rbx_klass = rbx_temp;
       __ movptr(rbx_klass, rcx_amh_argument); // this is a Class object!
       __ movptr(rbx_klass, Address(rbx_klass, java_lang_Class::klass_offset_in_bytes()));
+      __ load_heap_oop(rbx_klass, rcx_amh_argument); // this is a Class object!
+      __ load_heap_oop(rbx_klass, Address(rbx_klass, java_lang_Class::klass_offset_in_bytes()));
       Label ok_array_klass, bad_array_klass, bad_array_length;
 …
       // Arguments are spread.  Move to next method handle.
       UNPUSH_RSI_RDI;
       __ movptr(rcx_recv, rcx_mh_vmtarget);
+      __ load_heap_oop(rcx_recv, rcx_mh_vmtarget);
       __ jump_to_method_handle_entry(rcx_recv, rdx_temp);
       __ bind(bad_array_klass);
       UNPUSH_RSI_RDI;
+      __ pushptr(Address(rdx_array_klass, java_mirror_offset)); // required type
+      __ pushptr(vmarg);                // bad array
+      __ push((int)Bytecodes::_aaload); // who is complaining?
+      assert(!vmarg.uses(rarg2_required), "must be different registers");
+      __ movptr(rarg2_required, Address(rdx_array_klass, java_mirror_offset));  // required type
+      __ movptr(rarg1_actual,   vmarg);                                         // bad array
+      __ movl(  rarg0_code,     (int) Bytecodes::_aaload);                      // who is complaining?
       __ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));
       __ bind(bad_array_length);
       UNPUSH_RSI_RDI;
+      __ push(rcx_recv);        // AMH requiring a certain length
+      __ pushptr(vmarg);        // bad array
+      __ push((int)Bytecodes::_arraylength); // who is complaining?
+      assert(!vmarg.uses(rarg2_required), "must be different registers");
+      __ mov   (rarg2_required, rcx_recv);                       // AMH requiring a certain length
+      __ movptr(rarg1_actual,   vmarg);                          // bad array
+      __ movl(  rarg0_code,     (int) Bytecodes::_arraylength);  // who is complaining?
       __ jump(ExternalAddress(from_interpreted_entry(_raise_exception)));

trunk/openjdk/hotspot/src/cpu/x86/vm/nativeInst_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_nativeInst_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/handles.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "utilities/ostream.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Runtime1.hpp"
+#endif
 void NativeInstruction::wrote(int offset) {

trunk/openjdk/hotspot/src/cpu/x86/vm/nativeInst_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_NATIVEINST_X86_HPP
+#define CPU_X86_VM_NATIVEINST_X86_HPP
+#include "asm/assembler.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/icache.hpp"
+#include "runtime/os.hpp"
+#include "utilities/top.hpp"
 // We have interfaces for the following instructions:
 …
 #endif // AMD64
+}
+#endif // CPU_X86_VM_NATIVEINST_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/registerMap_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_REGISTERMAP_X86_HPP
+#define CPU_X86_VM_REGISTERMAP_X86_HPP
 // machine-dependent implemention for register maps
   friend class frame;
 …
   void pd_initialize() {}
   void pd_initialize_from(const RegisterMap* map) {}
+#endif // CPU_X86_VM_REGISTERMAP_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/register_definitions_x86.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_register_definitions_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "asm/register.hpp"
+#include "register_x86.hpp"
+#ifdef TARGET_ARCH_MODEL_x86_32
+# include "interp_masm_x86_32.hpp"
+#endif
+#ifdef TARGET_ARCH_MODEL_x86_64
+# include "interp_masm_x86_64.hpp"
+#endif
 REGISTER_DEFINITION(Register, noreg);

trunk/openjdk/hotspot/src/cpu/x86/vm/register_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_register_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "register_x86.hpp"
 #ifndef AMD64
 const int ConcreteRegisterImpl::max_gpr = RegisterImpl::number_of_registers;

trunk/openjdk/hotspot/src/cpu/x86/vm/register_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2000, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_REGISTER_X86_HPP
+#define CPU_X86_VM_REGISTER_X86_HPP
+#include "asm/register.hpp"
+#include "vm_version_x86.hpp"
 class VMRegImpl;
 …
 };
+#endif // CPU_X86_VM_REGISTER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/relocInfo_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+# include "incls/_precompiled.incl"
+# include "incls/_relocInfo_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.inline.hpp"
+#include "assembler_x86.inline.hpp"
+#include "code/relocInfo.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/safepoint.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/relocInfo_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_RELOCINFO_X86_HPP
+#define CPU_X86_VM_RELOCINFO_X86_HPP
   // machine-dependent parts of class relocInfo
  private:
 …
 #endif
   };
+#endif // CPU_X86_VM_RELOCINFO_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/runtime_x86_32.cpp

-              r278
+              r309
  */
+#include "precompiled.hpp"
+#ifdef COMPILER2
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "code/vmreg.hpp"
+#include "interpreter/interpreter.hpp"
+#include "nativeInst_x86.hpp"
+#include "opto/runtime.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "vmreg_x86.inline.hpp"
+#endif
-#include "incls/_precompiled.incl"
-#include "incls/_runtime_x86_32.cpp.incl"
 #define __ masm->

trunk/openjdk/hotspot/src/cpu/x86/vm/runtime_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2006, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "precompiled.hpp"
+#ifdef COMPILER2
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "code/vmreg.hpp"
+#include "interpreter/interpreter.hpp"
+#include "nativeInst_x86.hpp"
+#include "opto/runtime.hpp"
+#include "runtime/interfaceSupport.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "vmreg_x86.inline.hpp"
+#endif
 // This file should really contain the code for generating the OptoRuntime

trunk/openjdk/hotspot/src/cpu/x86/vm/sharedRuntime_x86_32.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_sharedRuntime_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "code/debugInfoRec.hpp"
+#include "code/icBuffer.hpp"
+#include "code/vtableStubs.hpp"
+#include "interpreter/interpreter.hpp"
+#include "oops/compiledICHolderOop.hpp"
+#include "prims/jvmtiRedefineClassesTrace.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/vframeArray.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Runtime1.hpp"
+#endif
+#ifdef COMPILER2
+#include "opto/runtime.hpp"
+#endif
 #define __ masm->

trunk/openjdk/hotspot/src/cpu/x86/vm/sharedRuntime_x86_64.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_sharedRuntime_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "code/debugInfoRec.hpp"
+#include "code/icBuffer.hpp"
+#include "code/vtableStubs.hpp"
+#include "interpreter/interpreter.hpp"
+#include "oops/compiledICHolderOop.hpp"
+#include "prims/jvmtiRedefineClassesTrace.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/vframeArray.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Runtime1.hpp"
+#endif
+#ifdef COMPILER2
+#include "opto/runtime.hpp"
+#endif
 DeoptimizationBlob *SharedRuntime::_deopt_blob;

trunk/openjdk/hotspot/src/cpu/x86/vm/stubGenerator_x86_32.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_stubGenerator_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "interpreter/interpreter.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/instanceOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubCodeGenerator.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "utilities/top.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "thread_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "thread_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "thread_windows.inline.hpp"
+#endif
+#ifdef COMPILER2
+#include "opto/runtime.hpp"
+#endif
 // Declaration and definition of StubGenerator (no .hpp file).

trunk/openjdk/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp

-              r278
+              r309
  */
+#include "incls/_precompiled.incl"
+#include "incls/_stubGenerator_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "interpreter/interpreter.hpp"
+#include "nativeInst_x86.hpp"
+#include "oops/instanceOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/handles.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubCodeGenerator.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "utilities/top.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "thread_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "thread_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "thread_windows.inline.hpp"
+#endif
+#ifdef COMPILER2
+#include "opto/runtime.hpp"
+#endif
 // Declaration and definition of StubGenerator (no .hpp file).
 …
     __ enter(); // required for proper stackwalking of RuntimeStub frame
-    checkcast_copy_entry  = __ pc();
-    BLOCK_COMMENT("Entry:");
 #ifdef ASSERT
     // caller guarantees that the arrays really are different
 …
 #endif //ASSERT
-    // allocate spill slots for r13, r14
-    enum {
-      saved_r13_offset,
-      saved_r14_offset,
-      saved_rbp_offset,
-      saved_rip_offset,
-      saved_rarg0_offset
-    };
-    __ subptr(rsp, saved_rbp_offset * wordSize);
-    __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
-    __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
                        // ckoff => rcx, ckval => r8
 …
 #ifdef _WIN64
     // last argument (#4) is on stack on Win64
+    const int ckval_offset = saved_rarg0_offset + 4;
+    __ movptr(ckval, Address(rsp, ckval_offset * wordSize));
+    __ movptr(ckval, Address(rsp, 6 * wordSize));
 #endif
+    // Caller of this entry point must set up the argument registers.
+    checkcast_copy_entry  = __ pc();
+    BLOCK_COMMENT("Entry:");
+    // allocate spill slots for r13, r14
+    enum {
+      saved_r13_offset,
+      saved_r14_offset,
+      saved_rbp_offset
+    };
+    __ subptr(rsp, saved_rbp_offset * wordSize);
+    __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
+    __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
     // check that int operands are properly extended to size_t
 …
     const Register dst        = c_rarg2;  // destination array oop
     const Register dst_pos    = c_rarg3;  // destination position
+    // elements count is on stack on Win64
+#ifdef _WIN64
+#define C_RARG4 Address(rsp, 6 * wordSize)
+#ifndef _WIN64
+    const Register length     = c_rarg4;
 #else
 #define C_RARG4 c_rarg4
+    const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
 #endif
 …
     const Register r11_length    = r11; // elements count to copy
     const Register r10_src_klass = r10; // array klass
-    const Register r9_dst_klass  = r9;  // dest array klass
     //  if (length < 0) return -1;
     __ movl(r11_length, C_RARG4);       // length (elements count, 32-bits value)
+    __ movl(r11_length, length);        // length (elements count, 32-bits value)
     __ testl(r11_length, r11_length);
     __ jccb(Assembler::negative, L_failed_0);
 …
 #ifdef ASSERT
     //  assert(src->klass() != NULL);
+    BLOCK_COMMENT("assert klasses not null");
+    { Label L1, L2;
+    {
+      BLOCK_COMMENT("assert klasses not null {");
+      Label L1, L2;
       __ testptr(r10_src_klass, r10_src_klass);
       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
 …
       __ stop("broken null klass");
       __ bind(L2);
       __ load_klass(r9_dst_klass, dst);
       __ cmpq(r9_dst_klass, 0);
+      __ load_klass(rax, dst);
+      __ cmpq(rax, 0);
       __ jcc(Assembler::equal, L1);     // this would be broken also
       BLOCK_COMMENT("assert done");
+      BLOCK_COMMENT("} assert klasses not null done");
+    }
 #endif
 …
     //
+    int lh_offset = klassOopDesc::header_size() * HeapWordSize +
+                    Klass::layout_helper_offset_in_bytes();
+    const int lh_offset = klassOopDesc::header_size() * HeapWordSize +
+                          Klass::layout_helper_offset_in_bytes();
+    // Handle objArrays completely differently...
+    const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
+    __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
+    __ jcc(Assembler::equal, L_objArray);
+    //  if (src->klass() != dst->klass()) return -1;
+    __ load_klass(rax, dst);
+    __ cmpq(r10_src_klass, rax);
+    __ jcc(Assembler::notEqual, L_failed);
     const Register rax_lh = rax;  // layout helper
     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
-    // Handle objArrays completely differently...
-    jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
-    __ cmpl(rax_lh, objArray_lh);
-    __ jcc(Assembler::equal, L_objArray);
-    //  if (src->klass() != dst->klass()) return -1;
-    __ load_klass(r9_dst_klass, dst);
-    __ cmpq(r10_src_klass, r9_dst_klass);
-    __ jcc(Assembler::notEqual, L_failed);
     //  if (!src->is_Array()) return -1;
 …
     // At this point, it is known to be a typeArray (array_tag 0x3).
 #ifdef ASSERT
+    { Label L;
+    {
+      BLOCK_COMMENT("assert primitive array {");
+      Label L;
       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
       __ jcc(Assembler::greaterEqual, L);
       __ stop("must be a primitive array");
       __ bind(L);
+      BLOCK_COMMENT("} assert primitive array done");
+    }
 #endif
 …
   __ BIND(L_copy_longs);
 #ifdef ASSERT
+    { Label L;
+    {
+      BLOCK_COMMENT("assert long copy {");
+      Label L;
       __ cmpl(rax_elsize, LogBytesPerLong);
       __ jcc(Assembler::equal, L);
       __ stop("must be long copy, but elsize is wrong");
       __ bind(L);
+      BLOCK_COMMENT("} assert long copy done");
+    }
 #endif
 …
     // objArrayKlass
   __ BIND(L_objArray);
     // live at this point:  r10_src_klass, src[_pos], dst[_pos]
+    // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
     Label L_plain_copy, L_checkcast_copy;
     //  test array classes for subtyping
     __ load_klass(r9_dst_klass, dst);
     __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality
+    __ load_klass(rax, dst);
+    __ cmpq(r10_src_klass, rax); // usual case is exact equality
     __ jcc(Assembler::notEqual, L_checkcast_copy);
 …
   __ BIND(L_checkcast_copy);
     // live at this point:  r10_src_klass, !r11_length
+    // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
+    {
-      // assert(r11_length == C_RARG4); // will reload from here
-      Register r11_dst_klass = r11;
-      __ load_klass(r11_dst_klass, dst);
       // Before looking at dst.length, make sure dst is also an objArray.
       __ cmpl(Address(r11_dst_klass, lh_offset), objArray_lh);
+      __ cmpl(Address(rax, lh_offset), objArray_lh);
       __ jcc(Assembler::notEqual, L_failed);
       // It is safe to examine both src.length and dst.length.
-#ifndef _WIN64
-      arraycopy_range_checks(src, src_pos, dst, dst_pos, C_RARG4,
-                             rax, L_failed);
-#else
-      __ movl(r11_length, C_RARG4);     // reload
       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
                              rax, L_failed);
+      const Register r11_dst_klass = r11;
       __ load_klass(r11_dst_klass, dst); // reload
-#endif
       // Marshal the base address arguments now, freeing registers.
 …
       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
       __ movl(count, C_RARG4);          // length (reloaded)
+      __ movl(count, length);           // length (reloaded)
       Register sco_temp = c_rarg3;      // this register is free now
       assert_different_registers(from, to, count, sco_temp,
 …
       // Generate the type check.
       int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
                         Klass::super_check_offset_offset_in_bytes());
+      const int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
+                              Klass::super_check_offset_offset_in_bytes());
       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
       assert_clean_int(sco_temp, rax);
 …
                        objArrayKlass::element_klass_offset_in_bytes());
       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
       __ movl(sco_temp,      Address(r11_dst_klass, sco_offset));
+      __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
       assert_clean_int(sco_temp, rax);
       // the checkcast_copy loop needs two extra arguments:
       assert(c_rarg3 == sco_temp, "#3 already in place");
+      __ movptr(C_RARG4, r11_dst_klass);  // dst.klass.element_klass
+      // Set up arguments for checkcast_copy_entry.
+      setup_arg_regs(4);
+      __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
       __ jump(RuntimeAddress(checkcast_copy_entry));
+    }
 …
     return start;
+  }
-#undef length_arg
   void generate_arraycopy_stubs() {

trunk/openjdk/hotspot/src/cpu/x86/vm/stubRoutines_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_stubRoutines_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/stubRoutines.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "thread_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "thread_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "thread_windows.inline.hpp"
+#endif
 // Implementation of the platform-specific part of StubRoutines - for

trunk/openjdk/hotspot/src/cpu/x86/vm/stubRoutines_x86_32.hpp

-              r278
+              r309
  */
+#ifndef CPU_X86_VM_STUBROUTINES_X86_32_HPP
+#define CPU_X86_VM_STUBROUTINES_X86_32_HPP
 // This file holds the platform specific parts of the StubRoutines
 // definition. See stubRoutines.hpp for a description on how to
 …
 // MethodHandles adapters
 enum method_handles_platform_dependent_constants {
   method_handles_adapters_code_size = 5000
+  method_handles_adapters_code_size = 10000
 };
 …
   static bool    returns_to_call_stub(address return_pc)     { return (return_pc == _call_stub_return_address) ||
                                                                        return_pc == x86::get_call_stub_compiled_return(); }
+#endif // CPU_X86_VM_STUBROUTINES_X86_32_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/stubRoutines_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_stubRoutines_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/stubRoutines.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "thread_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "thread_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "thread_windows.inline.hpp"
+#endif
 // Implementation of the platform-specific part of StubRoutines - for

trunk/openjdk/hotspot/src/cpu/x86/vm/stubRoutines_x86_64.hpp

-              r278
+              r309
  */
+#ifndef CPU_X86_VM_STUBROUTINES_X86_64_HPP
+#define CPU_X86_VM_STUBROUTINES_X86_64_HPP
 // This file holds the platform specific parts of the StubRoutines
 // definition. See stubRoutines.hpp for a description on how to
 …
 // MethodHandles adapters
 enum method_handles_platform_dependent_constants {
   method_handles_adapters_code_size = 13000
+  method_handles_adapters_code_size = 40000
 };
 …
+  }
 };
+#endif // CPU_X86_VM_STUBROUTINES_X86_64_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/templateInterpreterGenerator_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_TEMPLATEINTERPRETERGENERATOR_X86_HPP
+#define CPU_X86_VM_TEMPLATEINTERPRETERGENERATOR_X86_HPP
  protected:
 …
  // address generate_asm_interpreter_entry(bool synchronized);
+#endif // CPU_X86_VM_TEMPLATEINTERPRETERGENERATOR_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/templateInterpreter_x86.hpp

-              r278
+              r309
  */
+#ifndef CPU_X86_VM_TEMPLATEINTERPRETER_X86_HPP
+#define CPU_X86_VM_TEMPLATEINTERPRETER_X86_HPP
   protected:
 …
   const static int InterpreterCodeSize = 168 * 1024;
 #endif // AMD64
+#endif // CPU_X86_VM_TEMPLATEINTERPRETER_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/templateInterpreter_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_templateInterpreter_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "interpreter/bytecodeHistogram.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterGenerator.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "interpreter/templateTable.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/arguments.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
+#include "runtime/timer.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
 #define __ _masm->
 …
 //
 void InterpreterGenerator::generate_counter_incr(Label* overflow, Label* profile_method, Label* profile_method_continue) {
+  const Address invocation_counter(rbx, methodOopDesc::invocation_counter_offset() + InvocationCounter::counter_offset());
+  const Address backedge_counter  (rbx, methodOopDesc::backedge_counter_offset() + InvocationCounter::counter_offset());
+  if (ProfileInterpreter) { // %%% Merge this into methodDataOop
+    __ incrementl(Address(rbx,methodOopDesc::interpreter_invocation_counter_offset()));
+  }
+  // Update standard invocation counters
+  __ movl(rax, backedge_counter);               // load backedge counter
+  __ incrementl(rcx, InvocationCounter::count_increment);
+  __ andl(rax, InvocationCounter::count_mask_value);  // mask out the status bits
+  __ movl(invocation_counter, rcx);             // save invocation count
+  __ addl(rcx, rax);                            // add both counters
+  // profile_method is non-null only for interpreted method so
+  // profile_method != NULL == !native_call
+  // BytecodeInterpreter only calls for native so code is elided.
+  if (ProfileInterpreter && profile_method != NULL) {
+    // Test to see if we should create a method data oop
+  const Address invocation_counter(rbx, in_bytes(methodOopDesc::invocation_counter_offset()) +
+                                        in_bytes(InvocationCounter::counter_offset()));
+  // Note: In tiered we increment either counters in methodOop or in MDO depending if we're profiling or not.
+  if (TieredCompilation) {
+    int increment = InvocationCounter::count_increment;
+    int mask = ((1 << Tier0InvokeNotifyFreqLog)  - 1) << InvocationCounter::count_shift;
+    Label no_mdo, done;
+    if (ProfileInterpreter) {
+      // Are we profiling?
+      __ movptr(rax, Address(rbx, methodOopDesc::method_data_offset()));
+      __ testptr(rax, rax);
+      __ jccb(Assembler::zero, no_mdo);
+      // Increment counter in the MDO
+      const Address mdo_invocation_counter(rax, in_bytes(methodDataOopDesc::invocation_counter_offset()) +
+                                                in_bytes(InvocationCounter::counter_offset()));
+      __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
+      __ jmpb(done);
+    }
+    __ bind(no_mdo);
+    // Increment counter in methodOop (we don't need to load it, it's in rcx).
+    __ increment_mask_and_jump(invocation_counter, increment, mask, rcx, true, Assembler::zero, overflow);
+    __ bind(done);
+  } else {
+    const Address backedge_counter  (rbx, methodOopDesc::backedge_counter_offset() +
+                                          InvocationCounter::counter_offset());
+    if (ProfileInterpreter) { // %%% Merge this into methodDataOop
+      __ incrementl(Address(rbx,methodOopDesc::interpreter_invocation_counter_offset()));
+    }
+    // Update standard invocation counters
+    __ movl(rax, backedge_counter);               // load backedge counter
+    __ incrementl(rcx, InvocationCounter::count_increment);
+    __ andl(rax, InvocationCounter::count_mask_value);  // mask out the status bits
+    __ movl(invocation_counter, rcx);             // save invocation count
+    __ addl(rcx, rax);                            // add both counters
+    // profile_method is non-null only for interpreted method so
+    // profile_method != NULL == !native_call
+    // BytecodeInterpreter only calls for native so code is elided.
+    if (ProfileInterpreter && profile_method != NULL) {
+      // Test to see if we should create a method data oop
+      __ cmp32(rcx,
+               ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
+      __ jcc(Assembler::less, *profile_method_continue);
+      // if no method data exists, go to profile_method
+      __ test_method_data_pointer(rax, *profile_method);
+    }
     __ cmp32(rcx,
+             ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
+    __ jcc(Assembler::less, *profile_method_continue);
+    // if no method data exists, go to profile_method
+    __ test_method_data_pointer(rax, *profile_method);
+  }
+  __ cmp32(rcx,
+           ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
+  __ jcc(Assembler::aboveEqual, *overflow);
+             ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
+    __ jcc(Assembler::aboveEqual, *overflow);
+  }
+}
 …
       // We have decided to profile this method in the interpreter
       __ bind(profile_method);
+      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), rsi, true);
+      __ movptr(rbx, Address(rbp, method_offset));   // restore methodOop
+      __ movptr(rax, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax);
+      __ test_method_data_pointer(rax, profile_method_continue);
+      __ addptr(rax, in_bytes(methodDataOopDesc::data_offset()));
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rax);
+      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
+      __ set_method_data_pointer_for_bcp();
+      __ get_method(rbx);
       __ jmp(profile_method_continue);
+    }

trunk/openjdk/hotspot/src/cpu/x86/vm/templateInterpreter_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_interpreter_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "interpreter/bytecodeHistogram.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterGenerator.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "interpreter/templateTable.hpp"
+#include "oops/arrayOop.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/methodOop.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/jvmtiExport.hpp"
+#include "prims/jvmtiThreadState.hpp"
+#include "runtime/arguments.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
+#include "runtime/timer.hpp"
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
 #define __ _masm->
 …
         Label* profile_method,
         Label* profile_method_continue) {
+  const Address invocation_counter(rbx,
+                                   methodOopDesc::invocation_counter_offset() +
+  const Address invocation_counter(rbx, in_bytes(methodOopDesc::invocation_counter_offset()) +
+                                        in_bytes(InvocationCounter::counter_offset()));
+  // Note: In tiered we increment either counters in methodOop or in MDO depending if we're profiling or not.
+  if (TieredCompilation) {
+    int increment = InvocationCounter::count_increment;
+    int mask = ((1 << Tier0InvokeNotifyFreqLog)  - 1) << InvocationCounter::count_shift;
+    Label no_mdo, done;
+    if (ProfileInterpreter) {
+      // Are we profiling?
+      __ movptr(rax, Address(rbx, methodOopDesc::method_data_offset()));
+      __ testptr(rax, rax);
+      __ jccb(Assembler::zero, no_mdo);
+      // Increment counter in the MDO
+      const Address mdo_invocation_counter(rax, in_bytes(methodDataOopDesc::invocation_counter_offset()) +
+                                                in_bytes(InvocationCounter::counter_offset()));
+      __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
+      __ jmpb(done);
+    }
+    __ bind(no_mdo);
+    // Increment counter in methodOop (we don't need to load it, it's in ecx).
+    __ increment_mask_and_jump(invocation_counter, increment, mask, rcx, true, Assembler::zero, overflow);
+    __ bind(done);
+  } else {
+    const Address backedge_counter(rbx,
+                                   methodOopDesc::backedge_counter_offset() +
                                    InvocationCounter::counter_offset());
+  const Address backedge_counter(rbx,
+                                 methodOopDesc::backedge_counter_offset() +
+                                 InvocationCounter::counter_offset());
+  if (ProfileInterpreter) { // %%% Merge this into methodDataOop
+    __ incrementl(Address(rbx,
+                    methodOopDesc::interpreter_invocation_counter_offset()));
+  }
+  // Update standard invocation counters
+  __ movl(rax, backedge_counter); // load backedge counter
+  __ incrementl(rcx, InvocationCounter::count_increment);
+  __ andl(rax, InvocationCounter::count_mask_value); // mask out the
+                                                     // status bits
+  __ movl(invocation_counter, rcx); // save invocation count
+  __ addl(rcx, rax); // add both counters
+  // profile_method is non-null only for interpreted method so
+  // profile_method != NULL == !native_call
+  if (ProfileInterpreter && profile_method != NULL) {
+    // Test to see if we should create a method data oop
+    __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
+    __ jcc(Assembler::less, *profile_method_continue);
+    // if no method data exists, go to profile_method
+    __ test_method_data_pointer(rax, *profile_method);
+  }
+  __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
+  __ jcc(Assembler::aboveEqual, *overflow);
+    if (ProfileInterpreter) { // %%% Merge this into methodDataOop
+      __ incrementl(Address(rbx,
+                            methodOopDesc::interpreter_invocation_counter_offset()));
+    }
+    // Update standard invocation counters
+    __ movl(rax, backedge_counter);   // load backedge counter
+    __ incrementl(rcx, InvocationCounter::count_increment);
+    __ andl(rax, InvocationCounter::count_mask_value); // mask out the status bits
+    __ movl(invocation_counter, rcx); // save invocation count
+    __ addl(rcx, rax);                // add both counters
+    // profile_method is non-null only for interpreted method so
+    // profile_method != NULL == !native_call
+    if (ProfileInterpreter && profile_method != NULL) {
+      // Test to see if we should create a method data oop
+      __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
+      __ jcc(Assembler::less, *profile_method_continue);
+      // if no method data exists, go to profile_method
+      __ test_method_data_pointer(rax, *profile_method);
+    }
+    __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
+    __ jcc(Assembler::aboveEqual, *overflow);
+  }
+}
 …
     //
     __ mov(c_rarg0, r15_thread);
     __ mov(r12, rsp); // remember sp
+    __ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
     __ andptr(rsp, -16); // align stack as required by ABI
 …
     __ pusha(); // XXX only save smashed registers
     __ mov(r12, rsp); // remember sp
+    __ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
     __ andptr(rsp, -16); // align stack as required by ABI
 …
       // We have decided to profile this method in the interpreter
       __ bind(profile_method);
+      __ call_VM(noreg,
+                 CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method),
+                 r13, true);
+      __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop
+      __ movptr(rax, Address(rbx,
+                             in_bytes(methodOopDesc::method_data_offset())));
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
+                rax);
+      __ test_method_data_pointer(rax, profile_method_continue);
+      __ addptr(rax, in_bytes(methodDataOopDesc::data_offset()));
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
+              rax);
+      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
+      __ set_method_data_pointer_for_bcp();
+      __ get_method(rbx);
       __ jmp(profile_method_continue);
+    }
 …
   assert(Interpreter::trace_code(t->tos_in()) != NULL,
          "entry must have been generated");
   __ mov(r12, rsp); // remember sp
+  __ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
   __ andptr(rsp, -16); // align stack as required by ABI
   __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));

trunk/openjdk/hotspot/src/cpu/x86/vm/templateTable_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_templateTable_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "interpreter/templateTable.hpp"
+#include "memory/universe.inline.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
 #ifndef CC_INTERP
 …
     __ verify_oop(rax);
+  }
+  Label L_done, L_throw_exception;
+  const Register con_klass_temp = rcx;  // same as Rcache
+  __ movptr(con_klass_temp, Address(rax, oopDesc::klass_offset_in_bytes()));
+  __ cmpptr(con_klass_temp, ExternalAddress((address)Universe::systemObjArrayKlassObj_addr()));
+  __ jcc(Assembler::notEqual, L_done);
+  __ cmpl(Address(rax, arrayOopDesc::length_offset_in_bytes()), 0);
+  __ jcc(Assembler::notEqual, L_throw_exception);
+  __ xorptr(rax, rax);
+  __ jmp(L_done);
+  // Load the exception from the system-array which wraps it:
+  __ bind(L_throw_exception);
+  __ movptr(rax, Address(rax, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
+  __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
+  __ bind(L_done);
+}
 …
     __ jcc(Assembler::positive, dispatch); // count only if backward branch
+    // increment counter
+    __ movl(rax, Address(rcx, be_offset));        // load backedge counter
+    __ incrementl(rax, InvocationCounter::count_increment); // increment counter
+    __ movl(Address(rcx, be_offset), rax);        // store counter
+    __ movl(rax, Address(rcx, inv_offset));    // load invocation counter
+    __ andl(rax, InvocationCounter::count_mask_value);     // and the status bits
+    __ addl(rax, Address(rcx, be_offset));        // add both counters
+    if (ProfileInterpreter) {
+      // Test to see if we should create a method data oop
+      __ cmp32(rax,
+               ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
+      __ jcc(Assembler::less, dispatch);
+      // if no method data exists, go to profile method
+      __ test_method_data_pointer(rax, profile_method);
+      if (UseOnStackReplacement) {
+        // check for overflow against rbx, which is the MDO taken count
+        __ cmp32(rbx,
+                 ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+        __ jcc(Assembler::below, dispatch);
+        // When ProfileInterpreter is on, the backedge_count comes from the
+        // methodDataOop, which value does not get reset on the call to
+        // frequency_counter_overflow().  To avoid excessive calls to the overflow
+        // routine while the method is being compiled, add a second test to make
+        // sure the overflow function is called only once every overflow_frequency.
+        const int overflow_frequency = 1024;
+        __ andptr(rbx, overflow_frequency-1);
+        __ jcc(Assembler::zero, backedge_counter_overflow);
+    if (TieredCompilation) {
+      Label no_mdo;
+      int increment = InvocationCounter::count_increment;
+      int mask = ((1 << Tier0BackedgeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
+      if (ProfileInterpreter) {
+        // Are we profiling?
+        __ movptr(rbx, Address(rcx, in_bytes(methodOopDesc::method_data_offset())));
+        __ testptr(rbx, rbx);
+        __ jccb(Assembler::zero, no_mdo);
+        // Increment the MDO backedge counter
+        const Address mdo_backedge_counter(rbx, in_bytes(methodDataOopDesc::backedge_counter_offset()) +
+                                                in_bytes(InvocationCounter::counter_offset()));
+        __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
+                                   rax, false, Assembler::zero, &backedge_counter_overflow);
+        __ jmp(dispatch);
+      }
+      __ bind(no_mdo);
+      // Increment backedge counter in methodOop
+      __ increment_mask_and_jump(Address(rcx, be_offset), increment, mask,
+                                 rax, false, Assembler::zero, &backedge_counter_overflow);
     } else {
+      if (UseOnStackReplacement) {
+        // check for overflow against rax, which is the sum of the counters
+      // increment counter
+      __ movl(rax, Address(rcx, be_offset));        // load backedge counter
+      __ incrementl(rax, InvocationCounter::count_increment); // increment counter
+      __ movl(Address(rcx, be_offset), rax);        // store counter
+      __ movl(rax, Address(rcx, inv_offset));    // load invocation counter
+      __ andl(rax, InvocationCounter::count_mask_value);     // and the status bits
+      __ addl(rax, Address(rcx, be_offset));        // add both counters
+      if (ProfileInterpreter) {
+        // Test to see if we should create a method data oop
         __ cmp32(rax,
+                 ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+        __ jcc(Assembler::aboveEqual, backedge_counter_overflow);
+                 ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
+        __ jcc(Assembler::less, dispatch);
+        // if no method data exists, go to profile method
+        __ test_method_data_pointer(rax, profile_method);
+        if (UseOnStackReplacement) {
+          // check for overflow against rbx, which is the MDO taken count
+          __ cmp32(rbx,
+                   ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+          __ jcc(Assembler::below, dispatch);
+          // When ProfileInterpreter is on, the backedge_count comes from the
+          // methodDataOop, which value does not get reset on the call to
+          // frequency_counter_overflow().  To avoid excessive calls to the overflow
+          // routine while the method is being compiled, add a second test to make
+          // sure the overflow function is called only once every overflow_frequency.
+          const int overflow_frequency = 1024;
+          __ andptr(rbx, overflow_frequency-1);
+          __ jcc(Assembler::zero, backedge_counter_overflow);
+        }
+      } else {
+        if (UseOnStackReplacement) {
+          // check for overflow against rax, which is the sum of the counters
+          __ cmp32(rax,
+                   ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+          __ jcc(Assembler::aboveEqual, backedge_counter_overflow);
+        }
+      }
+    }
 …
       // Out-of-line code to allocate method data oop.
       __ bind(profile_method);
       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method), rsi);
+      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
       __ load_unsigned_byte(rbx, Address(rsi, 0));  // restore target bytecode
+      __ movptr(rcx, Address(rbp, method_offset));
+      __ movptr(rcx, Address(rcx, in_bytes(methodOopDesc::method_data_offset())));
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rcx);
+      __ test_method_data_pointer(rcx, dispatch);
+      // offset non-null mdp by MDO::data_offset() + IR::profile_method()
+      __ addptr(rcx, in_bytes(methodDataOopDesc::data_offset()));
+      __ addptr(rcx, rax);
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize), rcx);
+      __ set_method_data_pointer_for_bcp();
       __ jmp(dispatch);
+    }
 …
   // rax: CallSite object (f1)
   // rbx: unused (f2)
+  // rcx: receiver address
   // rdx: flags (unused)
+  Register rax_callsite      = rax;
+  Register rcx_method_handle = rcx;
   if (ProfileInterpreter) {
-    Label L;
     // %%% should make a type profile for any invokedynamic that takes a ref argument
     // profile this call
 …
+  }
   __ movptr(rcx, Address(rax, __ delayed_value(java_dyn_CallSite::target_offset_in_bytes, rcx)));
   __ null_check(rcx);
+  __ movptr(rcx_method_handle, Address(rax_callsite, __ delayed_value(java_dyn_CallSite::target_offset_in_bytes, rcx)));
+  __ null_check(rcx_method_handle);
   __ prepare_to_jump_from_interpreted();
   __ jump_to_method_handle_entry(rcx, rdx);
+  __ jump_to_method_handle_entry(rcx_method_handle, rdx);
+}
 …
     Universe::heap()->supports_inline_contig_alloc() && !CMSIncrementalMode;
+  const Register thread = rcx;
+  if (UseTLAB || allow_shared_alloc) {
+    __ get_thread(thread);
+  }
   if (UseTLAB) {
-    const Register thread = rcx;
-    __ get_thread(thread);
     __ movptr(rax, Address(thread, in_bytes(JavaThread::tlab_top_offset())));
     __ lea(rbx, Address(rax, rdx, Address::times_1));
 …
     // if someone beat us on the allocation, try again, otherwise continue
     __ jcc(Assembler::notEqual, retry);
+    __ incr_allocated_bytes(thread, rdx, 0);
+  }
 …
     __ jcc(Assembler::zero, initialize_header);
   // Initialize topmost object field, divide rdx by 8, check if odd and
   // test if zero.
+    // Initialize topmost object field, divide rdx by 8, check if odd and
+    // test if zero.
     __ xorl(rcx, rcx);    // use zero reg to clear memory (shorter code)
     __ shrl(rdx, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
   // rdx must have been multiple of 8
+    // rdx must have been multiple of 8
 #ifdef ASSERT
     // make sure rdx was multiple of 8

trunk/openjdk/hotspot/src/cpu/x86/vm/templateTable_x86_32.hpp

-              r278
+              r309
 /*
  * Copyright (c) 1998, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_TEMPLATETABLE_X86_32_HPP
+#define CPU_X86_VM_TEMPLATETABLE_X86_32_HPP
   static void prepare_invoke(Register method, Register index, int byte_no);
   static void invokevirtual_helper(Register index, Register recv,
 …
   static void index_check(Register array, Register index);
   static void index_check_without_pop(Register array, Register index);
+#endif // CPU_X86_VM_TEMPLATETABLE_X86_32_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/templateTable_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_templateTable_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interpreterRuntime.hpp"
+#include "interpreter/templateTable.hpp"
+#include "memory/universe.inline.hpp"
+#include "oops/methodDataOop.hpp"
+#include "oops/objArrayKlass.hpp"
+#include "oops/oop.inline.hpp"
+#include "prims/methodHandles.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/stubRoutines.hpp"
+#include "runtime/synchronizer.hpp"
 #ifndef CC_INTERP
 …
     __ verify_oop(rax);
+  }
+  Label L_done, L_throw_exception;
+  const Register con_klass_temp = rcx;  // same as cache
+  const Register array_klass_temp = rdx;  // same as index
+  __ movptr(con_klass_temp, Address(rax, oopDesc::klass_offset_in_bytes()));
+  __ lea(array_klass_temp, ExternalAddress((address)Universe::systemObjArrayKlassObj_addr()));
+  __ cmpptr(con_klass_temp, Address(array_klass_temp, 0));
+  __ jcc(Assembler::notEqual, L_done);
+  __ cmpl(Address(rax, arrayOopDesc::length_offset_in_bytes()), 0);
+  __ jcc(Assembler::notEqual, L_throw_exception);
+  __ xorptr(rax, rax);
+  __ jmp(L_done);
+  // Load the exception from the system-array which wraps it:
+  __ bind(L_throw_exception);
+  __ movptr(rax, Address(rax, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
+  __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
+  __ bind(L_done);
+}
 …
     __ testl(rdx, rdx);             // check if forward or backward branch
     __ jcc(Assembler::positive, dispatch); // count only if backward branch
+    // increment counter
+    __ movl(rax, Address(rcx, be_offset));        // load backedge counter
+    __ incrementl(rax, InvocationCounter::count_increment); // increment
+                                                            // counter
+    __ movl(Address(rcx, be_offset), rax);        // store counter
+    __ movl(rax, Address(rcx, inv_offset));    // load invocation counter
+    __ andl(rax, InvocationCounter::count_mask_value); // and the status bits
+    __ addl(rax, Address(rcx, be_offset));        // add both counters
+    if (ProfileInterpreter) {
+      // Test to see if we should create a method data oop
+      __ cmp32(rax,
+               ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
+      __ jcc(Assembler::less, dispatch);
+      // if no method data exists, go to profile method
+      __ test_method_data_pointer(rax, profile_method);
+      if (UseOnStackReplacement) {
+        // check for overflow against ebx which is the MDO taken count
+        __ cmp32(rbx,
+                 ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+        __ jcc(Assembler::below, dispatch);
+        // When ProfileInterpreter is on, the backedge_count comes
+        // from the methodDataOop, which value does not get reset on
+        // the call to frequency_counter_overflow().  To avoid
+        // excessive calls to the overflow routine while the method is
+        // being compiled, add a second test to make sure the overflow
+        // function is called only once every overflow_frequency.
+        const int overflow_frequency = 1024;
+        __ andl(rbx, overflow_frequency - 1);
+        __ jcc(Assembler::zero, backedge_counter_overflow);
+    if (TieredCompilation) {
+      Label no_mdo;
+      int increment = InvocationCounter::count_increment;
+      int mask = ((1 << Tier0BackedgeNotifyFreqLog) - 1) << InvocationCounter::count_shift;
+      if (ProfileInterpreter) {
+        // Are we profiling?
+        __ movptr(rbx, Address(rcx, in_bytes(methodOopDesc::method_data_offset())));
+        __ testptr(rbx, rbx);
+        __ jccb(Assembler::zero, no_mdo);
+        // Increment the MDO backedge counter
+        const Address mdo_backedge_counter(rbx, in_bytes(methodDataOopDesc::backedge_counter_offset()) +
+                                           in_bytes(InvocationCounter::counter_offset()));
+        __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
+                                   rax, false, Assembler::zero, &backedge_counter_overflow);
+        __ jmp(dispatch);
+      }
+      __ bind(no_mdo);
+      // Increment backedge counter in methodOop
+      __ increment_mask_and_jump(Address(rcx, be_offset), increment, mask,
+                                 rax, false, Assembler::zero, &backedge_counter_overflow);
     } else {
+      if (UseOnStackReplacement) {
+        // check for overflow against eax, which is the sum of the
+        // counters
+      // increment counter
+      __ movl(rax, Address(rcx, be_offset));        // load backedge counter
+      __ incrementl(rax, InvocationCounter::count_increment); // increment counter
+      __ movl(Address(rcx, be_offset), rax);        // store counter
+      __ movl(rax, Address(rcx, inv_offset));    // load invocation counter
+      __ andl(rax, InvocationCounter::count_mask_value); // and the status bits
+      __ addl(rax, Address(rcx, be_offset));        // add both counters
+      if (ProfileInterpreter) {
+        // Test to see if we should create a method data oop
         __ cmp32(rax,
+                 ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+        __ jcc(Assembler::aboveEqual, backedge_counter_overflow);
+                 ExternalAddress((address) &InvocationCounter::InterpreterProfileLimit));
+        __ jcc(Assembler::less, dispatch);
+        // if no method data exists, go to profile method
+        __ test_method_data_pointer(rax, profile_method);
+        if (UseOnStackReplacement) {
+          // check for overflow against ebx which is the MDO taken count
+          __ cmp32(rbx,
+                   ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+          __ jcc(Assembler::below, dispatch);
+          // When ProfileInterpreter is on, the backedge_count comes
+          // from the methodDataOop, which value does not get reset on
+          // the call to frequency_counter_overflow().  To avoid
+          // excessive calls to the overflow routine while the method is
+          // being compiled, add a second test to make sure the overflow
+          // function is called only once every overflow_frequency.
+          const int overflow_frequency = 1024;
+          __ andl(rbx, overflow_frequency - 1);
+          __ jcc(Assembler::zero, backedge_counter_overflow);
+        }
+      } else {
+        if (UseOnStackReplacement) {
+          // check for overflow against eax, which is the sum of the
+          // counters
+          __ cmp32(rax,
+                   ExternalAddress((address) &InvocationCounter::InterpreterBackwardBranchLimit));
+          __ jcc(Assembler::aboveEqual, backedge_counter_overflow);
+        }
+      }
+    }
 …
       // Out-of-line code to allocate method data oop.
       __ bind(profile_method);
+      __ call_VM(noreg,
+                 CAST_FROM_FN_PTR(address,
+                                  InterpreterRuntime::profile_method), r13);
+      __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
       __ load_unsigned_byte(rbx, Address(r13, 0));  // restore target bytecode
+      __ movptr(rcx, Address(rbp, method_offset));
+      __ movptr(rcx, Address(rcx,
+                             in_bytes(methodOopDesc::method_data_offset())));
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
+                rcx);
+      __ test_method_data_pointer(rcx, dispatch);
+      // offset non-null mdp by MDO::data_offset() + IR::profile_method()
+      __ addptr(rcx, in_bytes(methodDataOopDesc::data_offset()));
+      __ addptr(rcx, rax);
+      __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
+                rcx);
+      __ set_method_data_pointer_for_bcp();
       __ jmp(dispatch);
+    }
 …
     __ get_cache_entry_pointer_at_bcp(c_rarg2, rcx, 1);
     __ verify_oop(rax);
     __ mov(r12, rax);  // save object pointer before call_VM() clobbers it
+    __ push_ptr(rax);  // save object pointer before call_VM() clobbers it
     __ mov(c_rarg1, rax);
     // c_rarg1: object pointer copied above
 …
                                 InterpreterRuntime::post_field_access),
                c_rarg1, c_rarg2);
+    __ mov(rax, r12); // restore object pointer
+    __ reinit_heapbase();
+    __ pop_ptr(rax); // restore object pointer
     __ bind(L1);
+  }
 …
                                          Register recv,
                                          Register flags) {
+  // Uses temporary registers rax, rdx  assert_different_registers(index, recv, rax, rdx);
+  // Uses temporary registers rax, rdx
+  assert_different_registers(index, recv, rax, rdx);
   // Test for an invoke of a final method
 …
   // rdx: flags (unused)
+  Register rax_callsite      = rax;
+  Register rcx_method_handle = rcx;
   if (ProfileInterpreter) {
-    Label L;
     // %%% should make a type profile for any invokedynamic that takes a ref argument
     // profile this call
 …
+  }
   __ movptr(rcx, Address(rax, __ delayed_value(java_dyn_CallSite::target_offset_in_bytes, rcx)));
   __ null_check(rcx);
+  __ load_heap_oop(rcx_method_handle, Address(rax_callsite, __ delayed_value(java_dyn_CallSite::target_offset_in_bytes, rcx)));
+  __ null_check(rcx_method_handle);
   __ prepare_to_jump_from_interpreted();
   __ jump_to_method_handle_entry(rcx, rdx);
+  __ jump_to_method_handle_entry(rcx_method_handle, rdx);
+}
 …
     // if someone beat us on the allocation, try again, otherwise continue
     __ jcc(Assembler::notEqual, retry);
+    __ incr_allocated_bytes(r15_thread, rdx, 0);
+  }
 …
   __ jcc(Assembler::equal, quicked);
   __ push(atos); // save receiver for result, and for GC
-  __ mov(r12, rcx); // save rcx XXX
   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
-  __ movq(rcx, r12); // restore rcx XXX
-  __ reinit_heapbase();
   __ pop_ptr(rdx); // restore receiver
   __ jmpb(resolved);
 …
   __ push(atos); // save receiver for result, and for GC
-  __ mov(r12, rcx); // save rcx
   call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
-  __ movq(rcx, r12); // restore rcx
-  __ reinit_heapbase();
   __ pop_ptr(rdx); // restore receiver
+  __ verify_oop(rdx);
   __ load_klass(rdx, rdx);
   __ jmpb(resolved);

trunk/openjdk/hotspot/src/cpu/x86/vm/templateTable_x86_64.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_TEMPLATETABLE_X86_64_HPP
+#define CPU_X86_VM_TEMPLATETABLE_X86_64_HPP
   static void prepare_invoke(Register method, Register index, int byte_no);
   static void invokevirtual_helper(Register index, Register recv,
 …
   static void index_check(Register array, Register index);
   static void index_check_without_pop(Register array, Register index);
+#endif // CPU_X86_VM_TEMPLATETABLE_X86_64_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/vmStructs_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2001, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_VMSTRUCTS_X86_HPP
+#define CPU_X86_VM_VMSTRUCTS_X86_HPP
 // These are the CPU-specific fields, types and integer
 …
   /* in vmStructs_<os>_<cpu>.hpp's VM_LONG_CONSTANTS_OS_CPU macro (and must  */
   /* be present there)                                                       */
+#endif // CPU_X86_VM_VMSTRUCTS_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/vm_version_x86.cpp

-              r278
+              r309
  */
+# include "incls/_precompiled.incl"
+# include "incls/_vm_version_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "assembler_x86.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/java.hpp"
+#include "runtime/stubCodeGenerator.hpp"
+#include "vm_version_x86.hpp"
+#ifdef TARGET_OS_FAMILY_linux
+# include "os_linux.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_solaris
+# include "os_solaris.inline.hpp"
+#endif
+#ifdef TARGET_OS_FAMILY_windows
+# include "os_windows.inline.hpp"
+#endif
 …
   char buf[256];
   jio_snprintf(buf, sizeof(buf), "(%u cores per cpu, %u threads per core) family %d model %d stepping %d%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
+  jio_snprintf(buf, sizeof(buf), "(%u cores per cpu, %u threads per core) family %d model %d stepping %d%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
                cores_per_cpu(), threads_per_core(),
                cpu_family(), _model, _stepping,
 …
                (supports_popcnt() ? ", popcnt" : ""),
                (supports_mmx_ext() ? ", mmxext" : ""),
+               (supports_3dnow()   ? ", 3dnow"  : ""),
+               (supports_3dnow2()  ? ", 3dnowext" : ""),
+               (supports_3dnow_prefetch() ? ", 3dnowpref" : ""),
                (supports_lzcnt()   ? ", lzcnt": ""),
                (supports_sse4a()   ? ", sse4a": ""),
 …
+      }
+    }
+    if( FLAG_IS_DEFAULT(UseSSE42Intrinsics) ) {
+      if( supports_sse4_2() && UseSSE >= 4 ) {
+        UseSSE42Intrinsics = true;
+      }
+    }
     // Use count leading zeros count instruction if available.
 …
+      }
+    }
+    // some defaults for AMD family 15h
+    if ( cpu_family() == 0x15 ) {
+      // On family 15h processors default is no sw prefetch
+      if (FLAG_IS_DEFAULT(AllocatePrefetchStyle)) {
+        AllocatePrefetchStyle = 0;
+      }
+      // Also, if some other prefetch style is specified, default instruction type is PREFETCHW
+      if (FLAG_IS_DEFAULT(AllocatePrefetchInstr)) {
+        AllocatePrefetchInstr = 3;
+      }
+      // On family 15h processors use XMM and UnalignedLoadStores for Array Copy
+      if( FLAG_IS_DEFAULT(UseXMMForArrayCopy) ) {
+        UseXMMForArrayCopy = true;
+      }
+      if( FLAG_IS_DEFAULT(UseUnalignedLoadStores) && UseXMMForArrayCopy ) {
+        UseUnalignedLoadStores = true;
+      }
+    }
+  }
 …
   if( ReadPrefetchInstr < 0 ) ReadPrefetchInstr = 0;
   if( ReadPrefetchInstr > 3 ) ReadPrefetchInstr = 3;
   if( ReadPrefetchInstr == 3 && !supports_3dnow() ) ReadPrefetchInstr = 0;
   if( !supports_sse() && supports_3dnow() ) ReadPrefetchInstr = 3;
+  if( ReadPrefetchInstr == 3 && !supports_3dnow_prefetch() ) ReadPrefetchInstr = 0;
+  if( !supports_sse() && supports_3dnow_prefetch() ) ReadPrefetchInstr = 3;
   if( AllocatePrefetchInstr < 0 ) AllocatePrefetchInstr = 0;
   if( AllocatePrefetchInstr > 3 ) AllocatePrefetchInstr = 3;
   if( AllocatePrefetchInstr == 3 && !supports_3dnow() ) AllocatePrefetchInstr=0;
   if( !supports_sse() && supports_3dnow() ) AllocatePrefetchInstr = 3;
+  if( AllocatePrefetchInstr == 3 && !supports_3dnow_prefetch() ) AllocatePrefetchInstr=0;
+  if( !supports_sse() && supports_3dnow_prefetch() ) AllocatePrefetchInstr = 3;
   // Allocation prefetch settings
 …
                   logical_processors_per_package());
     tty->print_cr("UseSSE=%d",UseSSE);
     tty->print("Allocation: ");
     if (AllocatePrefetchStyle <= 0 || UseSSE == 0 && !supports_3dnow()) {
       tty->print_cr("no prefetching");
+    tty->print("Allocation");
+    if (AllocatePrefetchStyle <= 0 || UseSSE == 0 && !supports_3dnow_prefetch()) {
+      tty->print_cr(": no prefetching");
     } else {
+      if (UseSSE == 0 && supports_3dnow()) {
+      tty->print(" prefetching: ");
+      if (UseSSE == 0 && supports_3dnow_prefetch()) {
         tty->print("PREFETCHW");
       } else if (UseSSE >= 1) {
 …
     vm_exit_during_initialization("Unable to allocate getPsrInfo_stub");
+  }
+  CodeBuffer c(stub_blob->instructions_begin(),
+               stub_blob->instructions_size());
+  CodeBuffer c(stub_blob);
   VM_Version_StubGenerator g(&c);
   getPsrInfo_stub = CAST_TO_FN_PTR(getPsrInfo_stub_t,

trunk/openjdk/hotspot/src/cpu/x86/vm/vm_version_x86.hpp

-              r278
+              r309
  */
+#ifndef CPU_X86_VM_VM_VERSION_X86_HPP
+#define CPU_X86_VM_VM_VERSION_X86_HPP
+#include "runtime/globals_extension.hpp"
+#include "runtime/vm_version.hpp"
 class VM_Version : public Abstract_VM_Version {
 public:
 …
      CPU_HT     = (1 << 3),
      CPU_MMX    = (1 << 4),
+     CPU_3DNOW  = (1 << 5), // 3DNow comes from cpuid 0x80000001 (EDX)
+     CPU_3DNOW_PREFETCH  = (1 << 5), // Processor supports 3dnow prefetch and prefetchw instructions
+                                     // may not necessarily support other 3dnow instructions
      CPU_SSE    = (1 << 6),
      CPU_SSE2   = (1 << 7),
 …
     if (_cpuid_info.std_cpuid1_edx.bits.cmov != 0)
       result |= CPU_CMOV;
     if (_cpuid_info.std_cpuid1_edx.bits.fxsr != 0 || is_amd() &&
         _cpuid_info.ext_cpuid1_edx.bits.fxsr != 0)
+    if (_cpuid_info.std_cpuid1_edx.bits.fxsr != 0 || (is_amd() &&
+        _cpuid_info.ext_cpuid1_edx.bits.fxsr != 0))
       result |= CPU_FXSR;
     // HT flag is set for multi-core processors also.
     if (threads_per_core() > 1)
       result |= CPU_HT;
     if (_cpuid_info.std_cpuid1_edx.bits.mmx != 0 || is_amd() &&
         _cpuid_info.ext_cpuid1_edx.bits.mmx != 0)
+    if (_cpuid_info.std_cpuid1_edx.bits.mmx != 0 || (is_amd() &&
+        _cpuid_info.ext_cpuid1_edx.bits.mmx != 0))
       result |= CPU_MMX;
     if (_cpuid_info.std_cpuid1_edx.bits.sse != 0)
 …
     // AMD features.
     if (is_amd()) {
+      if (_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0)
+        result |= CPU_3DNOW;
+      if ((_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0) ||
+          (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0))
+        result |= CPU_3DNOW_PREFETCH;
       if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0)
         result |= CPU_LZCNT;
 …
   // AMD features
   //
   static bool supports_3dnow()    { return (_cpuFeatures & CPU_3DNOW) != 0; }
+  static bool supports_3dnow_prefetch()    { return (_cpuFeatures & CPU_3DNOW_PREFETCH) != 0; }
   static bool supports_mmx_ext()  { return is_amd() && _cpuid_info.ext_cpuid1_edx.bits.mmx_amd != 0; }
-  static bool supports_3dnow2()   { return is_amd() && _cpuid_info.ext_cpuid1_edx.bits.tdnow2 != 0; }
   static bool supports_lzcnt()    { return (_cpuFeatures & CPU_LZCNT) != 0; }
   static bool supports_sse4a()    { return (_cpuFeatures & CPU_SSE4A) != 0; }
+  // Intel Core and newer cpus have fast IDIV instruction (excluding Atom).
+  static bool has_fast_idiv()     { return is_intel() && cpu_family() == 6 &&
+                                           supports_sse3() && _model != 0x1C; }
   static bool supports_compare_and_exchange() { return true; }
 …
+  }
 };
+#endif // CPU_X86_VM_VM_VERSION_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/vmreg_x86.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2006, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_vmreg_x86.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "code/vmreg.hpp"

trunk/openjdk/hotspot/src/cpu/x86/vm/vmreg_x86.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2006, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#ifndef CPU_X86_VM_VMREG_X86_HPP
+#define CPU_X86_VM_VMREG_X86_HPP
   bool is_Register();
   Register as_Register();
 …
   bool is_XMMRegister();
   XMMRegister as_XMMRegister();
+#endif // CPU_X86_VM_VMREG_X86_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/vmreg_x86.inline.hpp

-              r278
+              r309
 /*
  * Copyright (c) 2006, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
+ *
  */
+#ifndef CPU_X86_VM_VMREG_X86_INLINE_HPP
+#define CPU_X86_VM_VMREG_X86_INLINE_HPP
 inline VMReg RegisterImpl::as_VMReg() {
 …
   return is_even(value());
+}
+#endif // CPU_X86_VM_VMREG_X86_INLINE_HPP

trunk/openjdk/hotspot/src/cpu/x86/vm/vtableStubs_x86_32.cpp

-              r278
+              r309
 /*
  * Copyright (c) 1997, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_vtableStubs_x86_32.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "code/vtableStubs.hpp"
+#include "interp_masm_x86_32.hpp"
+#include "memory/resourceArea.hpp"
+#include "oops/instanceKlass.hpp"
+#include "oops/klassVtable.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifdef COMPILER2
+#include "opto/runtime.hpp"
+#endif
 // machine-dependent part of VtableStubs: create VtableStub of correct size and

trunk/openjdk/hotspot/src/cpu/x86/vm/vtableStubs_x86_64.cpp

-              r278
+              r309
 /*
  * Copyright (c) 2003, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
 …
  */
+#include "incls/_precompiled.incl"
+#include "incls/_vtableStubs_x86_64.cpp.incl"
+#include "precompiled.hpp"
+#include "asm/assembler.hpp"
+#include "assembler_x86.inline.hpp"
+#include "code/vtableStubs.hpp"
+#include "interp_masm_x86_64.hpp"
+#include "memory/resourceArea.hpp"
+#include "oops/instanceKlass.hpp"
+#include "oops/klassVtable.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "vmreg_x86.inline.hpp"
+#ifdef COMPILER2
+#include "opto/runtime.hpp"
+#endif
 // machine-dependent part of VtableStubs: create VtableStub of correct size and
 …
   } else {
     // Itable stub size
     return (DebugVtables ? 512 : 72) + (CountCompiledCalls ? 13 : 0) +
+    return (DebugVtables ? 512 : 74) + (CountCompiledCalls ? 13 : 0) +
            (UseCompressedOops ? 32 : 0);  // 2 leaqs
+  }

trunk/openjdk/hotspot/src/cpu/x86/vm/x86_32.ad

-              r278
+              r309
 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);
+  *(cbuf.code_end()) = c;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+  cbuf.insts()->emit_int8(c);
+}
 …
 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
   unsigned char c = (unsigned char)( f1 | f2 );
+  *(cbuf.code_end()) = c;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+  cbuf.insts()->emit_int8(c);
+}
 // EMIT_OPCODE()
 void emit_opcode(CodeBuffer &cbuf, int code) {
+  *(cbuf.code_end()) = (unsigned char)code;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+  cbuf.insts()->emit_int8((unsigned char) code);
+}
 // EMIT_OPCODE() w/ relocation information
 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {
   cbuf.relocate(cbuf.inst_mark() + offset, reloc);
+  cbuf.relocate(cbuf.insts_mark() + offset, reloc);
   emit_opcode(cbuf, code);
+}
 …
 // EMIT_D8()
 void emit_d8(CodeBuffer &cbuf, int d8) {
+  *(cbuf.code_end()) = (unsigned char)d8;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+  cbuf.insts()->emit_int8((unsigned char) d8);
+}
 // EMIT_D16()
 void emit_d16(CodeBuffer &cbuf, int d16) {
+  *((short *)(cbuf.code_end())) = d16;
+  cbuf.set_code_end(cbuf.code_end() + 2);
+  cbuf.insts()->emit_int16(d16);
+}
 // EMIT_D32()
 void emit_d32(CodeBuffer &cbuf, int d32) {
+  *((int *)(cbuf.code_end())) = d32;
+  cbuf.set_code_end(cbuf.code_end() + 4);
+  cbuf.insts()->emit_int32(d32);
+}
 …
 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,
         int format) {
+  cbuf.relocate(cbuf.inst_mark(), reloc, format);
+  *((int *)(cbuf.code_end())) = d32;
+  cbuf.set_code_end(cbuf.code_end() + 4);
+  cbuf.relocate(cbuf.insts_mark(), reloc, format);
+  cbuf.insts()->emit_int32(d32);
+}
 …
+  }
 #endif
+  cbuf.relocate(cbuf.inst_mark(), rspec, format);
+  *((int *)(cbuf.code_end())) = d32;
+  cbuf.set_code_end(cbuf.code_end() + 4);
+  cbuf.relocate(cbuf.insts_mark(), rspec, format);
+  cbuf.insts()->emit_int32(d32);
+}
 …
 //=============================================================================
+const bool Matcher::constant_table_absolute_addressing = true;
+const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
+void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
+  // Empty encoding
+}
+uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
+  return 0;
+}
+#ifndef PRODUCT
+void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
+  st->print("# MachConstantBaseNode (empty encoding)");
+}
+#endif
+//=============================================================================
 #ifndef PRODUCT
 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
 …
     emit_d32(cbuf, framesize);
+  }
   C->set_frame_complete(cbuf.code_end() - cbuf.code_begin());
+  C->set_frame_complete(cbuf.insts_size());
 #ifdef ASSERT
 …
   if( do_polling() && C->is_method_compilation() ) {
     cbuf.relocate(cbuf.code_end(), relocInfo::poll_return_type, 0);
+    cbuf.relocate(cbuf.insts_end(), relocInfo::poll_return_type, 0);
     emit_opcode(cbuf,0x85);
     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX
 …
   // jmp -1
   address mark = cbuf.inst_mark();  // get mark within main instrs section
   // Note that the code buffer's inst_mark is always relative to insts.
+  address mark = cbuf.insts_mark();  // get mark within main instrs section
+  // Note that the code buffer's insts_mark is always relative to insts.
   // That's why we must use the macroassembler to generate a stub.
   MacroAssembler _masm(&cbuf);
 …
   __ end_a_stub();
   // Update current stubs pointer and restore code_end.
+  // Update current stubs pointer and restore insts_end.
+}
 // size of call stub, compiled java to interpretor
 …
   MacroAssembler masm(&cbuf);
 #ifdef ASSERT
   uint code_size = cbuf.code_size();
+  uint insts_size = cbuf.insts_size();
 #endif
   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
 …
   masm.nop(nops_cnt);
   assert(cbuf.code_size() - code_size == size(ra_), "checking code size of inline cache node");
+  assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");
+}
 …
 int emit_exception_handler(CodeBuffer& cbuf) {
   // Note that the code buffer's inst_mark is always relative to insts.
+  // Note that the code buffer's insts_mark is always relative to insts.
   // That's why we must use the macroassembler to generate a handler.
   MacroAssembler _masm(&cbuf);
 …
   if (base == NULL)  return 0;  // CodeBuffer::expand failed
   int offset = __ offset();
   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->instructions_begin()));
+  __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
   __ end_a_stub();
 …
 int emit_deopt_handler(CodeBuffer& cbuf) {
   // Note that the code buffer's inst_mark is always relative to insts.
+  // Note that the code buffer's insts_mark is always relative to insts.
   // That's why we must use the macroassembler to generate a handler.
   MacroAssembler _masm(&cbuf);
 …
-static void emit_double_constant(CodeBuffer& cbuf, double x) {
-  int mark = cbuf.insts()->mark_off();
-  MacroAssembler _masm(&cbuf);
-  address double_address = __ double_constant(x);
-  cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
-  emit_d32_reloc(cbuf,
-                 (int)double_address,
-                 internal_word_Relocation::spec(double_address),
-                 RELOC_DISP32);
+}
-static void emit_float_constant(CodeBuffer& cbuf, float x) {
-  int mark = cbuf.insts()->mark_off();
-  MacroAssembler _masm(&cbuf);
-  address float_address = __ float_constant(x);
-  cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
-  emit_d32_reloc(cbuf,
-                 (int)float_address,
-                 internal_word_Relocation::spec(float_address),
-                 RELOC_DISP32);
+}
 const bool Matcher::match_rule_supported(int opcode) {
   if (!has_match_rule(opcode))
 …
 int Matcher::regnum_to_fpu_offset(int regnum) {
   return regnum - 32; // The FP registers are in the second chunk
+}
-bool is_positive_zero_float(jfloat f) {
-  return jint_cast(f) == jint_cast(0.0F);
+}
-bool is_positive_one_float(jfloat f) {
-  return jint_cast(f) == jint_cast(1.0F);
+}
-bool is_positive_zero_double(jdouble d) {
-  return jlong_cast(d) == jlong_cast(0.0);
+}
-bool is_positive_one_double(jdouble d) {
-  return jlong_cast(d) == jlong_cast(1.0);
+}
 …
+}
+bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
+  // Use hardware integer DIV instruction when
+  // it is faster than a code which use multiply.
+  // Only when constant divisor fits into 32 bit
+  // (min_jint is excluded to get only correct
+  // positive 32 bit values from negative).
+  return VM_Version::has_fast_idiv() &&
+         (divisor == (int)divisor && divisor != min_jint);
+}
 // Register for DIVI projection of divmodI
 RegMask Matcher::divI_proj_mask() {
 …
       return true;
+    }
+  }
+  if (opc == Op_ConL && (n->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
+    return true;
+  }
   return false;
 …
   enc_class Lbl (label labl) %{ // JMP, CALL
     Label *l = $labl$$label;
     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size()+4)) : 0);
+    emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size()+4)) : 0);
   %}
   enc_class LblShort (label labl) %{ // JMP, CALL
     Label *l = $labl$$label;
     int disp = l ? (l->loc_pos() - (cbuf.code_size()+1)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size()+1)) : 0;
     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
     emit_d8(cbuf, disp);
 …
     $$$emit8$primary;
     emit_cc(cbuf, $secondary, $cop$$cmpcode);
     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size()+4)) : 0);
+    emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size()+4)) : 0);
   %}
 …
     Label *l = $labl$$label;
     emit_cc(cbuf, $primary, $cop$$cmpcode);
     int disp = l ? (l->loc_pos() - (cbuf.code_size()+1)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size()+1)) : 0;
     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
     emit_d8(cbuf, disp);
 …
   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf
     // This is the instruction starting address for relocation info.
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     // CALL directly to the runtime
     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
+    emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
                 runtime_call_Relocation::spec(), RELOC_IMM32 );
 …
   enc_class pre_call_FPU %{
     // If method sets FPU control word restore it here
     debug_only(int off0 = cbuf.code_size());
+    debug_only(int off0 = cbuf.insts_size());
     if( Compile::current()->in_24_bit_fp_mode() ) {
       MacroAssembler masm(&cbuf);
       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
+    }
     debug_only(int off1 = cbuf.code_size());
+    debug_only(int off1 = cbuf.insts_size());
     assert(off1 - off0 == pre_call_FPU_size(), "correct size prediction");
   %}
 …
   enc_class preserve_SP %{
     debug_only(int off0 = cbuf.code_size());
+    debug_only(int off0 = cbuf.insts_size());
     MacroAssembler _masm(&cbuf);
     // RBP is preserved across all calls, even compiled calls.
     // Use it to preserve RSP in places where the callee might change the SP.
     __ movptr(rbp_mh_SP_save, rsp);
     debug_only(int off1 = cbuf.code_size());
+    debug_only(int off1 = cbuf.insts_size());
     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
   %}
 …
     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
     // who we intended to call.
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     if ( !_method ) {
       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
+      emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
                      runtime_call_Relocation::spec(), RELOC_IMM32 );
     } else if(_optimized_virtual) {
       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
+      emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );
     } else {
       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
+      emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
                      static_call_Relocation::spec(), RELOC_IMM32 );
+    }
 …
     // Generate  "Mov EAX,0x00", placeholder instruction to load oop-info
     // emit_call_dynamic_prologue( cbuf );
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xB8 + EAX_enc);        // mov    EAX,-1
     emit_d32_reloc(cbuf, (int)Universe::non_oop_word(), oop_Relocation::spec_for_immediate(), RELOC_IMM32);
     address  virtual_call_oop_addr = cbuf.inst_mark();
+    address  virtual_call_oop_addr = cbuf.insts_mark();
     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
     // who we intended to call.
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
+    emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
                 virtual_call_Relocation::spec(virtual_call_oop_addr), RELOC_IMM32 );
   %}
 …
     // CALL *[EAX+in_bytes(methodOopDesc::from_compiled_code_entry_point_offset())]
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte
 …
 //
 //     // CALL to interpreter.
 //     cbuf.set_inst_mark();
+//     cbuf.set_insts_mark();
 //     $$$emit8$primary;
 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.code_end()) - 4),
+//     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.insts_end()) - 4),
 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );
 //   %}
 …
       emit_d32(cbuf, src_con);
+    }
-  %}
-  enc_class LdImmD (immD src) %{    // Load Immediate
-    if( is_positive_zero_double($src$$constant)) {
-      // FLDZ
-      emit_opcode(cbuf,0xD9);
-      emit_opcode(cbuf,0xEE);
-    } else if( is_positive_one_double($src$$constant)) {
-      // FLD1
-      emit_opcode(cbuf,0xD9);
-      emit_opcode(cbuf,0xE8);
-    } else {
-      emit_opcode(cbuf,0xDD);
-      emit_rm(cbuf, 0x0, 0x0, 0x5);
-      emit_double_constant(cbuf, $src$$constant);
+    }
-  %}
-  enc_class LdImmF (immF src) %{    // Load Immediate
-    if( is_positive_zero_float($src$$constant)) {
-      emit_opcode(cbuf,0xD9);
-      emit_opcode(cbuf,0xEE);
-    } else if( is_positive_one_float($src$$constant)) {
-      emit_opcode(cbuf,0xD9);
-      emit_opcode(cbuf,0xE8);
-    } else {
-      $$$emit8$primary;
-      // Load immediate does not have a zero or sign extended version
-      // for 8-bit immediates
-      // First load to TOS, then move to dst
-      emit_rm(cbuf, 0x0, 0x0, 0x5);
-      emit_float_constant(cbuf, $src$$constant);
+    }
-  %}
-  enc_class LdImmX (regX dst, immXF con) %{    // Load Immediate
-    emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
-    emit_float_constant(cbuf, $con$$constant);
-  %}
-  enc_class LdImmXD (regXD dst, immXD con) %{    // Load Immediate
-    emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
-    emit_double_constant(cbuf, $con$$constant);
-  %}
-  enc_class load_conXD (regXD dst, immXD con) %{ // Load double constant
-    // UseXmmLoadAndClearUpper ? movsd(dst, con) : movlpd(dst, con)
-    emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
-    emit_opcode(cbuf, 0x0F);
-    emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
-    emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
-    emit_double_constant(cbuf, $con$$constant);
-  %}
-  enc_class Opc_MemImm_F(immF src) %{
-    cbuf.set_inst_mark();
-    $$$emit8$primary;
-    emit_rm(cbuf, 0x0, $secondary, 0x5);
-    emit_float_constant(cbuf, $src$$constant);
   %}
 …
   enc_class set_instruction_start( ) %{
     cbuf.set_inst_mark();            // Mark start of opcode for reloc info in mem operand
+    cbuf.set_insts_mark();            // Mark start of opcode for reloc info in mem operand
   %}
 …
     emit_opcode( cbuf, 0x8B ); // Move
     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
+    emit_d8(cbuf,$primary);
+    emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
+    emit_d8(cbuf,$cnt$$constant-32);
+    if( $cnt$$constant > 32 ) { // Shift, if not by zero
+      emit_d8(cbuf,$primary);
+      emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
+      emit_d8(cbuf,$cnt$$constant-32);
+    }
     emit_d8(cbuf,$primary);
     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));
 …
       emit_d8( cbuf, 0xC0-1+$src$$reg );
+    }
     cbuf.set_inst_mark();       // Mark start of opcode for reloc info in mem operand
+    cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
     emit_opcode(cbuf,$primary);
     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
 …
     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
     // AND $tmp,$y
     cbuf.set_inst_mark();       // Mark start of opcode for reloc info in mem operand
+    cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
     emit_opcode(cbuf,0x23);
     int reg_encoding = tmpReg;
 …
     emit_opcode(cbuf,               0x50+$src2$$reg  );
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);       // Call into runtime
     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Restore stack
     emit_opcode(cbuf, 0x83); // add  SP, #framesize
 …
     emit_opcode(cbuf,               0x50+$src2$$reg  );
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);       // Call into runtime
     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Restore stack
     emit_opcode(cbuf, 0x83); // add  SP, #framesize
 …
          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
          if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) {
+         if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
             // prefetchw [eax + Offset(_owner)-2]
             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
 …
          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
          if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) {
+         if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
             // prefetchw [eax + Offset(_owner)-2]
             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
 …
       masm.get_thread (boxReg) ;
       if ((EmitSync & 4096) && VM_Version::supports_3dnow() && os::is_MP()) {
+      if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
         // prefetchw [ebx + Offset(_owner)-2]
         masm.prefetchw(Address(rbx, ObjectMonitor::owner_offset_in_bytes()-2));
 …
   enc_class enc_rethrow() %{
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xE9);        // jmp    entry
     emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.code_end())-4,
+    emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.insts_end())-4,
                    runtime_call_Relocation::spec(), RELOC_IMM32 );
   %}
 …
     emit_d8    (cbuf,0xC0-1+$src$$reg );
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);       // Call into runtime
     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Carry on here...
   %}
 …
     emit_d8    (cbuf,0xC0-1+$src$$reg );
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);       // Call into runtime
     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Carry on here...
   %}
 …
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);       // Call into runtime
     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Carry on here...
   %}
 …
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);      // Call into runtime
     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Carry on here...
   %}
 …
     // CALL directly to the runtime
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf,0xE8);       // Call into runtime
     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
+    emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
     // Carry on here...
 …
   enc_class enc_storeL_volatile( memory mem, stackSlotL src ) %{
     store_to_stackslot( cbuf, 0x0DF, 0x05, $src$$disp );
     cbuf.set_inst_mark();            // Mark start of FIST in case $mem has an oop
+    cbuf.set_insts_mark();            // Mark start of FIST in case $mem has an oop
     emit_opcode(cbuf,0xDF);
     int rm_byte_opcode = 0x07;
 …
       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
+    }
     cbuf.set_inst_mark();            // Mark start of MOVSD in case $mem has an oop
+    cbuf.set_insts_mark();            // Mark start of MOVSD in case $mem has an oop
     { // MOVSD $mem,$tmp ! atomic long store
       emit_opcode(cbuf,0xF2);
 …
       emit_rm(cbuf, 0x3, $tmp$$reg, $tmp2$$reg);
+    }
     cbuf.set_inst_mark();            // Mark start of MOVSD in case $mem has an oop
+    cbuf.set_insts_mark();            // Mark start of MOVSD in case $mem has an oop
     { // MOVSD $mem,$tmp ! atomic long store
       emit_opcode(cbuf,0xF2);
 …
   enc_class Safepoint_Poll() %{
     cbuf.relocate(cbuf.inst_mark(), relocInfo::poll_type, 0);
+    cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_type, 0);
     emit_opcode(cbuf,0x85);
     emit_rm (cbuf, 0x0, 0x7, 0x5);
 …
 %}
 // Double Immediate
+// Double Immediate one
 operand immD1() %{
   predicate( UseSSE<=1 && n->getd() == 1.0 );
 …
 // Float Immediate zero
 operand immF0() %{
+  predicate( UseSSE == 0 && n->getf() == 0.0 );
+  predicate(UseSSE == 0 && n->getf() == 0.0F);
+  match(ConF);
+  op_cost(5);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+// Float Immediate one
+operand immF1() %{
+  predicate(UseSSE == 0 && n->getf() == 1.0F);
   match(ConF);
 …
 // The instruction usage is guarded by predicate in operand immF().
 instruct loadConF(regF dst, immF src) %{
   match(Set dst src);
+instruct loadConF(regF dst, immF con) %{
+  match(Set dst con);
   ins_cost(125);
+  format %{ "FLD_S  ST,$src\n\t"
+  format %{ "FLD_S  ST,[$constantaddress]\t# load from constant table: float=$con\n\t"
             "FSTP   $dst" %}
+  opcode(0xD9, 0x00);       /* D9 /0 */
+  ins_encode(LdImmF(src), Pop_Reg_F(dst) );
+  ins_pipe( fpu_reg_con );
+  ins_encode %{
+    __ fld_s($constantaddress($con));
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_con);
+%}
+// The instruction usage is guarded by predicate in operand immF0().
+instruct loadConF0(regF dst, immF0 con) %{
+  match(Set dst con);
+  ins_cost(125);
+  format %{ "FLDZ   ST\n\t"
+            "FSTP   $dst" %}
+  ins_encode %{
+    __ fldz();
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_con);
+%}
+// The instruction usage is guarded by predicate in operand immF1().
+instruct loadConF1(regF dst, immF1 con) %{
+  match(Set dst con);
+  ins_cost(125);
+  format %{ "FLD1   ST\n\t"
+            "FSTP   $dst" %}
+  ins_encode %{
+    __ fld1();
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_con);
 %}
 …
   match(Set dst con);
   ins_cost(125);
+  format %{ "MOVSS  $dst,[$con]" %}
+  ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), LdImmX(dst, con));
+  ins_pipe( pipe_slow );
+  format %{ "MOVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ movflt($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   ins_cost(100);
   format %{ "XORPS  $dst,$dst\t# float 0.0" %}
+  ins_encode( Opcode(0x0F), Opcode(0x57), RegReg(dst,dst));
+  ins_pipe( pipe_slow );
+  ins_encode %{
+    __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
+  %}
+  ins_pipe(pipe_slow);
 %}
 // The instruction usage is guarded by predicate in operand immD().
 instruct loadConD(regD dst, immD src) %{
   match(Set dst src);
+instruct loadConD(regD dst, immD con) %{
+  match(Set dst con);
   ins_cost(125);
   format %{ "FLD_D  ST,$src\n\t"
+  format %{ "FLD_D  ST,[$constantaddress]\t# load from constant table: double=$con\n\t"
             "FSTP   $dst" %}
+  ins_encode(LdImmD(src), Pop_Reg_D(dst) );
+  ins_pipe( fpu_reg_con );
+  ins_encode %{
+    __ fld_d($constantaddress($con));
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_con);
+%}
+// The instruction usage is guarded by predicate in operand immD0().
+instruct loadConD0(regD dst, immD0 con) %{
+  match(Set dst con);
+  ins_cost(125);
+  format %{ "FLDZ   ST\n\t"
+            "FSTP   $dst" %}
+  ins_encode %{
+    __ fldz();
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_con);
+%}
+// The instruction usage is guarded by predicate in operand immD1().
+instruct loadConD1(regD dst, immD1 con) %{
+  match(Set dst con);
+  ins_cost(125);
+  format %{ "FLD1   ST\n\t"
+            "FSTP   $dst" %}
+  ins_encode %{
+    __ fld1();
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_con);
 %}
 …
   match(Set dst con);
   ins_cost(125);
+  format %{ "MOVSD  $dst,[$con]" %}
+  ins_encode(load_conXD(dst, con));
+  ins_pipe( pipe_slow );
+  format %{ "MOVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ movdbl($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
 instruct prefetchr0( memory mem ) %{
   predicate(UseSSE==0 && !VM_Version::supports_3dnow());
+  predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
   match(PrefetchRead mem);
   ins_cost(0);
 …
 instruct prefetchr( memory mem ) %{
   predicate(UseSSE==0 && VM_Version::supports_3dnow() || ReadPrefetchInstr==3);
+  predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch() || ReadPrefetchInstr==3);
   match(PrefetchRead mem);
   ins_cost(100);
 …
 instruct prefetchw0( memory mem ) %{
   predicate(UseSSE==0 && !VM_Version::supports_3dnow());
+  predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
   match(PrefetchWrite mem);
   ins_cost(0);
 …
 instruct prefetchw( memory mem ) %{
   predicate(UseSSE==0 && VM_Version::supports_3dnow() || AllocatePrefetchInstr==3);
+  predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch() || AllocatePrefetchInstr==3);
   match( PrefetchWrite mem );
   ins_cost(100);
 …
 %}
+// Divide Register Long (no special case since divisor != -1)
+instruct divL_eReg_imm32( eADXRegL dst, immL32 imm, eRegI tmp, eRegI tmp2, eFlagsReg cr ) %{
+  match(Set dst (DivL dst imm));
+  effect( TEMP tmp, TEMP tmp2, KILL cr );
+  ins_cost(1000);
+  format %{ "MOV    $tmp,abs($imm) # ldiv EDX:EAX,$imm\n\t"
+            "XOR    $tmp2,$tmp2\n\t"
+            "CMP    $tmp,EDX\n\t"
+            "JA,s   fast\n\t"
+            "MOV    $tmp2,EAX\n\t"
+            "MOV    EAX,EDX\n\t"
+            "MOV    EDX,0\n\t"
+            "JLE,s  pos\n\t"
+            "LNEG   EAX : $tmp2\n\t"
+            "DIV    $tmp # unsigned division\n\t"
+            "XCHG   EAX,$tmp2\n\t"
+            "DIV    $tmp\n\t"
+            "LNEG   $tmp2 : EAX\n\t"
+            "JMP,s  done\n"
+    "pos:\n\t"
+            "DIV    $tmp\n\t"
+            "XCHG   EAX,$tmp2\n"
+    "fast:\n\t"
+            "DIV    $tmp\n"
+    "done:\n\t"
+            "MOV    EDX,$tmp2\n\t"
+            "NEG    EDX:EAX # if $imm < 0" %}
+  ins_encode %{
+    int con = (int)$imm$$constant;
+    assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
+    int pcon = (con > 0) ? con : -con;
+    Label Lfast, Lpos, Ldone;
+    __ movl($tmp$$Register, pcon);
+    __ xorl($tmp2$$Register,$tmp2$$Register);
+    __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
+    __ jccb(Assembler::above, Lfast); // result fits into 32 bit
+    __ movl($tmp2$$Register, $dst$$Register); // save
+    __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
+    __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
+    __ jccb(Assembler::lessEqual, Lpos); // result is positive
+    // Negative dividend.
+    // convert value to positive to use unsigned division
+    __ lneg($dst$$Register, $tmp2$$Register);
+    __ divl($tmp$$Register);
+    __ xchgl($dst$$Register, $tmp2$$Register);
+    __ divl($tmp$$Register);
+    // revert result back to negative
+    __ lneg($tmp2$$Register, $dst$$Register);
+    __ jmpb(Ldone);
+    __ bind(Lpos);
+    __ divl($tmp$$Register); // Use unsigned division
+    __ xchgl($dst$$Register, $tmp2$$Register);
+    // Fallthrow for final divide, tmp2 has 32 bit hi result
+    __ bind(Lfast);
+    // fast path: src is positive
+    __ divl($tmp$$Register); // Use unsigned division
+    __ bind(Ldone);
+    __ movl(HIGH_FROM_LOW($dst$$Register),$tmp2$$Register);
+    if (con < 0) {
+      __ lneg(HIGH_FROM_LOW($dst$$Register), $dst$$Register);
+    }
+  %}
+  ins_pipe( pipe_slow );
+%}
+// Remainder Register Long (remainder fit into 32 bits)
+instruct modL_eReg_imm32( eADXRegL dst, immL32 imm, eRegI tmp, eRegI tmp2, eFlagsReg cr ) %{
+  match(Set dst (ModL dst imm));
+  effect( TEMP tmp, TEMP tmp2, KILL cr );
+  ins_cost(1000);
+  format %{ "MOV    $tmp,abs($imm) # lrem EDX:EAX,$imm\n\t"
+            "CMP    $tmp,EDX\n\t"
+            "JA,s   fast\n\t"
+            "MOV    $tmp2,EAX\n\t"
+            "MOV    EAX,EDX\n\t"
+            "MOV    EDX,0\n\t"
+            "JLE,s  pos\n\t"
+            "LNEG   EAX : $tmp2\n\t"
+            "DIV    $tmp # unsigned division\n\t"
+            "MOV    EAX,$tmp2\n\t"
+            "DIV    $tmp\n\t"
+            "NEG    EDX\n\t"
+            "JMP,s  done\n"
+    "pos:\n\t"
+            "DIV    $tmp\n\t"
+            "MOV    EAX,$tmp2\n"
+    "fast:\n\t"
+            "DIV    $tmp\n"
+    "done:\n\t"
+            "MOV    EAX,EDX\n\t"
+            "SAR    EDX,31\n\t" %}
+  ins_encode %{
+    int con = (int)$imm$$constant;
+    assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
+    int pcon = (con > 0) ? con : -con;
+    Label  Lfast, Lpos, Ldone;
+    __ movl($tmp$$Register, pcon);
+    __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
+    __ jccb(Assembler::above, Lfast); // src is positive and result fits into 32 bit
+    __ movl($tmp2$$Register, $dst$$Register); // save
+    __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
+    __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
+    __ jccb(Assembler::lessEqual, Lpos); // result is positive
+    // Negative dividend.
+    // convert value to positive to use unsigned division
+    __ lneg($dst$$Register, $tmp2$$Register);
+    __ divl($tmp$$Register);
+    __ movl($dst$$Register, $tmp2$$Register);
+    __ divl($tmp$$Register);
+    // revert remainder back to negative
+    __ negl(HIGH_FROM_LOW($dst$$Register));
+    __ jmpb(Ldone);
+    __ bind(Lpos);
+    __ divl($tmp$$Register);
+    __ movl($dst$$Register, $tmp2$$Register);
+    __ bind(Lfast);
+    // fast path: src is positive
+    __ divl($tmp$$Register);
+    __ bind(Ldone);
+    __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
+    __ sarl(HIGH_FROM_LOW($dst$$Register), 31); // result sign
+  %}
+  ins_pipe( pipe_slow );
+%}
 // Integer Shift Instructions
 // Shift Left by one
 …
 %}
 instruct addD_reg_imm1(regD dst, immD1 src) %{
+instruct addD_reg_imm1(regD dst, immD1 con) %{
   predicate(UseSSE<=1);
   match(Set dst (AddD dst src));
+  match(Set dst (AddD dst con));
   ins_cost(125);
   format %{ "FLD1\n\t"
             "DADDp  $dst,ST" %}
+  opcode(0xDE, 0x00);
+  ins_encode( LdImmD(src),
+              OpcP, RegOpc(dst) );
+  ins_pipe( fpu_reg );
+%}
+instruct addD_reg_imm(regD dst, immD src) %{
+  ins_encode %{
+    __ fld1();
+    __ faddp($dst$$reg);
+  %}
+  ins_pipe(fpu_reg);
+%}
+instruct addD_reg_imm(regD dst, immD con) %{
   predicate(UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
   match(Set dst (AddD dst src));
+  match(Set dst (AddD dst con));
   ins_cost(200);
   format %{ "FLD_D  [$src]\n\t"
+  format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
             "DADDp  $dst,ST" %}
+  opcode(0xDE, 0x00);       /* DE /0 */
+  ins_encode( LdImmD(src),
+              OpcP, RegOpc(dst));
+  ins_pipe( fpu_reg_mem );
+  ins_encode %{
+    __ fld_d($constantaddress($con));
+    __ faddp($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_mem);
 %}
 …
   match(Set dst (RoundDouble (AddD src con)));
   ins_cost(200);
   format %{ "FLD_D  [$con]\n\t"
+  format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
             "DADD   ST,$src\n\t"
             "FSTP_D $dst\t# D-round" %}
+  opcode(0xD8, 0x00);       /* D8 /0 */
+  ins_encode( LdImmD(con),
+              OpcP, RegOpc(src), Pop_Mem_D(dst));
+  ins_pipe( fpu_mem_reg_con );
+  ins_encode %{
+    __ fld_d($constantaddress($con));
+    __ fadd($src$$reg);
+    __ fstp_d(Address(rsp, $dst$$disp));
+  %}
+  ins_pipe(fpu_mem_reg_con);
 %}
 …
   predicate(UseSSE>=2);
   match(Set dst (AddD dst con));
+  format %{ "ADDSD  $dst,[$con]" %}
+  ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), LdImmXD(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "ADDSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ addsd($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   predicate(UseSSE>=2);
   match(Set dst (SubD dst con));
+  format %{ "SUBSD  $dst,[$con]" %}
+  ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), LdImmXD(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "SUBSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ subsd($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   predicate(UseSSE>=2);
   match(Set dst (MulD dst con));
+  format %{ "MULSD  $dst,[$con]" %}
+  ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), LdImmXD(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "MULSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ mulsd($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   predicate(UseSSE>=2);
   match(Set dst (DivD dst con));
+  format %{ "DIVSD  $dst,[$con]" %}
+  ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), LdImmXD(dst, con));
+  ins_pipe( pipe_slow );
+  format %{ "DIVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ divsd($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
 %}
 instruct mulD_reg_imm(regD dst, immD src) %{
+instruct mulD_reg_imm(regD dst, immD con) %{
   predicate( UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
   match(Set dst (MulD dst src));
+  match(Set dst (MulD dst con));
   ins_cost(200);
   format %{ "FLD_D  [$src]\n\t"
+  format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
             "DMULp  $dst,ST" %}
+  opcode(0xDE, 0x1); /* DE /1 */
+  ins_encode( LdImmD(src),
+              OpcP, RegOpc(dst) );
+  ins_pipe( fpu_reg_mem );
+  ins_encode %{
+    __ fld_d($constantaddress($con));
+    __ fmulp($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_mem);
 %}
 …
   predicate(UseSSE>=1);
   match(Set dst (AddF dst con));
+  format %{ "ADDSS  $dst,[$con]" %}
+  ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), LdImmX(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "ADDSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ addss($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   predicate(UseSSE>=1);
   match(Set dst (SubF dst con));
+  format %{ "SUBSS  $dst,[$con]" %}
+  ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), LdImmX(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "SUBSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ subss($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   predicate(UseSSE>=1);
   match(Set dst (MulF dst con));
+  format %{ "MULSS  $dst,[$con]" %}
+  ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), LdImmX(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "MULSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ mulss($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
   predicate(UseSSE>=1);
   match(Set dst (DivF dst con));
+  format %{ "DIVSS  $dst,[$con]" %}
+  ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), LdImmX(dst, con) );
+  ins_pipe( pipe_slow );
+  format %{ "DIVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ divss($dst$$XMMRegister, $constantaddress($con));
+  %}
+  ins_pipe(pipe_slow);
 %}
 …
 // Spill to obtain 24-bit precision
 instruct addF24_reg_imm(stackSlotF dst, regF src1, immF src2) %{
+instruct addF24_reg_imm(stackSlotF dst, regF src, immF con) %{
   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
   match(Set dst (AddF src1 src2));
   format %{ "FLD    $src1\n\t"
             "FADD   $src2\n\t"
+  match(Set dst (AddF src con));
+  format %{ "FLD    $src\n\t"
+            "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
             "FSTP_S $dst"  %}
+  opcode(0xD8, 0x00);       /* D8 /0 */
+  ins_encode( Push_Reg_F(src1),
+              Opc_MemImm_F(src2),
+              Pop_Mem_F(dst));
+  ins_pipe( fpu_mem_reg_con );
+  ins_encode %{
+    __ fld_s($src$$reg - 1);  // FLD ST(i-1)
+    __ fadd_s($constantaddress($con));
+    __ fstp_s(Address(rsp, $dst$$disp));
+  %}
+  ins_pipe(fpu_mem_reg_con);
 %}
 //
 // This instruction does not round to 24-bits
 instruct addF_reg_imm(regF dst, regF src1, immF src2) %{
+instruct addF_reg_imm(regF dst, regF src, immF con) %{
   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
+  match(Set dst (AddF src1 src2));
+  format %{ "FLD    $src1\n\t"
+            "FADD   $src2\n\t"
+            "FSTP_S $dst"  %}
+  opcode(0xD8, 0x00);       /* D8 /0 */
+  ins_encode( Push_Reg_F(src1),
+              Opc_MemImm_F(src2),
+              Pop_Reg_F(dst));
+  ins_pipe( fpu_reg_reg_con );
+  match(Set dst (AddF src con));
+  format %{ "FLD    $src\n\t"
+            "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
+            "FSTP   $dst"  %}
+  ins_encode %{
+    __ fld_s($src$$reg - 1);  // FLD ST(i-1)
+    __ fadd_s($constantaddress($con));
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_reg_con);
 %}
 …
 // Spill to obtain 24-bit precision
 instruct mulF24_reg_imm(stackSlotF dst, regF src1, immF src2) %{
+instruct mulF24_reg_imm(stackSlotF dst, regF src, immF con) %{
   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
+  match(Set dst (MulF src1 src2));
+  format %{ "FMULc $dst,$src1,$src2" %}
+  opcode(0xD8, 0x1);  /* D8 /1*/
+  ins_encode( Push_Reg_F(src1),
+              Opc_MemImm_F(src2),
+              Pop_Mem_F(dst));
+  ins_pipe( fpu_mem_reg_con );
+  match(Set dst (MulF src con));
+  format %{ "FLD    $src\n\t"
+            "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
+            "FSTP_S $dst"  %}
+  ins_encode %{
+    __ fld_s($src$$reg - 1);  // FLD ST(i-1)
+    __ fmul_s($constantaddress($con));
+    __ fstp_s(Address(rsp, $dst$$disp));
+  %}
+  ins_pipe(fpu_mem_reg_con);
 %}
 //
 // This instruction does not round to 24-bits
 instruct mulF_reg_imm(regF dst, regF src1, immF src2) %{
+instruct mulF_reg_imm(regF dst, regF src, immF con) %{
   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
+  match(Set dst (MulF src1 src2));
+  format %{ "FMULc $dst. $src1, $src2" %}
+  opcode(0xD8, 0x1);  /* D8 /1*/
+  ins_encode( Push_Reg_F(src1),
+              Opc_MemImm_F(src2),
+              Pop_Reg_F(dst));
+  ins_pipe( fpu_reg_reg_con );
+  match(Set dst (MulF src con));
+  format %{ "FLD    $src\n\t"
+            "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
+            "FSTP   $dst"  %}
+  ins_encode %{
+    __ fld_s($src$$reg - 1);  // FLD ST(i-1)
+    __ fmul_s($constantaddress($con));
+    __ fstp_d($dst$$reg);
+  %}
+  ins_pipe(fpu_reg_reg_con);
 %}
 …
 %}
 instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eBXRegI cnt2,
                         eAXRegI result, regXD tmp1, regXD tmp2, eFlagsReg cr) %{
+instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
+                        eAXRegI result, regXD tmp1, eFlagsReg cr) %{
   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
+  effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
+  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
   ins_encode %{
     __ string_compare($str1$$Register, $str2$$Register,
                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
                       $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+                      $tmp1$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 …
   match(Jump switch_val);
   ins_cost(350);
+  format %{  "JMP    [table_base](,$switch_val,1)\n\t" %}
+  format %{  "JMP    [$constantaddress](,$switch_val,1)\n\t" %}
   ins_encode %{
-    address table_base  = __ address_table_constant(_index2label);
     // Jump to Address(table_base + switch_reg)
-    InternalAddress table(table_base);
     Address index(noreg, $switch_val$$Register, Address::times_1);
     __ jump(ArrayAddress(table, index));
+    __ jump(ArrayAddress($constantaddress, index));
   %}
   ins_pc_relative(1);
 …
     if ($cop$$cmpcode == Assembler::notEqual) {
        // the two jumps 6 bytes apart so the jump distances are too
        parity_disp = l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0;
+       parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
     } else if ($cop$$cmpcode == Assembler::equal) {
        parity_disp = 6;
 …
     $$$emit8$primary;
     emit_cc(cbuf, $secondary, $cop$$cmpcode);
     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
     emit_d32(cbuf, disp);
   %}
 …
     int parity_disp = -1;
     if ($cop$$cmpcode == Assembler::notEqual) {
       parity_disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
+      parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
     } else if ($cop$$cmpcode == Assembler::equal) {
       parity_disp = 2;
 …
     emit_d8(cbuf, parity_disp);
     emit_cc(cbuf, $primary, $cop$$cmpcode);
     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
     emit_d8(cbuf, disp);
     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");

trunk/openjdk/hotspot/src/cpu/x86/vm/x86_64.ad

-              r278
+              r309
 // EMIT_RM()
+void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3)
+{
+void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
+  *(cbuf.code_end()) = c;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+  cbuf.insts()->emit_int8(c);
+}
 // EMIT_CC()
+void emit_cc(CodeBuffer &cbuf, int f1, int f2)
+{
+void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
   unsigned char c = (unsigned char) (f1 | f2);
+  *(cbuf.code_end()) = c;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+  cbuf.insts()->emit_int8(c);
+}
 // EMIT_OPCODE()
+void emit_opcode(CodeBuffer &cbuf, int code)
+{
+  *(cbuf.code_end()) = (unsigned char) code;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+void emit_opcode(CodeBuffer &cbuf, int code) {
+  cbuf.insts()->emit_int8((unsigned char) code);
+}
 …
                  int code, relocInfo::relocType reloc, int offset, int format)
+{
   cbuf.relocate(cbuf.inst_mark() + offset, reloc, format);
+  cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
   emit_opcode(cbuf, code);
+}
 // EMIT_D8()
+void emit_d8(CodeBuffer &cbuf, int d8)
+{
+  *(cbuf.code_end()) = (unsigned char) d8;
+  cbuf.set_code_end(cbuf.code_end() + 1);
+void emit_d8(CodeBuffer &cbuf, int d8) {
+  cbuf.insts()->emit_int8((unsigned char) d8);
+}
 // EMIT_D16()
+void emit_d16(CodeBuffer &cbuf, int d16)
+{
+  *((short *)(cbuf.code_end())) = d16;
+  cbuf.set_code_end(cbuf.code_end() + 2);
+void emit_d16(CodeBuffer &cbuf, int d16) {
+  cbuf.insts()->emit_int16(d16);
+}
 // EMIT_D32()
+void emit_d32(CodeBuffer &cbuf, int d32)
+{
+  *((int *)(cbuf.code_end())) = d32;
+  cbuf.set_code_end(cbuf.code_end() + 4);
+void emit_d32(CodeBuffer &cbuf, int d32) {
+  cbuf.insts()->emit_int32(d32);
+}
 // EMIT_D64()
+void emit_d64(CodeBuffer &cbuf, int64_t d64)
+{
+  *((int64_t*) (cbuf.code_end())) = d64;
+  cbuf.set_code_end(cbuf.code_end() + 8);
+void emit_d64(CodeBuffer &cbuf, int64_t d64) {
+  cbuf.insts()->emit_int64(d64);
+}
 …
+{
   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
+  cbuf.relocate(cbuf.inst_mark(), reloc, format);
+  *((int*) (cbuf.code_end())) = d32;
+  cbuf.set_code_end(cbuf.code_end() + 4);
+  cbuf.relocate(cbuf.insts_mark(), reloc, format);
+  cbuf.insts()->emit_int32(d32);
+}
 // emit 32 bit value and construct relocation entry from RelocationHolder
+void emit_d32_reloc(CodeBuffer& cbuf,
+                    int d32,
+                    RelocationHolder const& rspec,
+                    int format)
+{
+void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 #ifdef ASSERT
   if (rspec.reloc()->type() == relocInfo::oop_type &&
 …
+  }
 #endif
+  cbuf.relocate(cbuf.inst_mark(), rspec, format);
+  *((int* )(cbuf.code_end())) = d32;
+  cbuf.set_code_end(cbuf.code_end() + 4);
+  cbuf.relocate(cbuf.insts_mark(), rspec, format);
+  cbuf.insts()->emit_int32(d32);
+}
 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
   address next_ip = cbuf.code_end() + 4;
+  address next_ip = cbuf.insts_end() + 4;
   emit_d32_reloc(cbuf, (int) (addr - next_ip),
                  external_word_Relocation::spec(addr),
 …
 // emit 64 bit value and construct relocation entry from relocInfo::relocType
+void emit_d64_reloc(CodeBuffer& cbuf,
+                    int64_t d64,
+                    relocInfo::relocType reloc,
+                    int format)
+{
+  cbuf.relocate(cbuf.inst_mark(), reloc, format);
+  *((int64_t*) (cbuf.code_end())) = d64;
+  cbuf.set_code_end(cbuf.code_end() + 8);
+void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
+  cbuf.relocate(cbuf.insts_mark(), reloc, format);
+  cbuf.insts()->emit_int64(d64);
+}
 // emit 64 bit value and construct relocation entry from RelocationHolder
+void emit_d64_reloc(CodeBuffer& cbuf,
+                    int64_t d64,
+                    RelocationHolder const& rspec,
+                    int format)
+{
+void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 #ifdef ASSERT
   if (rspec.reloc()->type() == relocInfo::oop_type &&
 …
+  }
 #endif
+  cbuf.relocate(cbuf.inst_mark(), rspec, format);
+  *((int64_t*) (cbuf.code_end())) = d64;
+  cbuf.set_code_end(cbuf.code_end() + 8);
+  cbuf.relocate(cbuf.insts_mark(), rspec, format);
+  cbuf.insts()->emit_int64(d64);
+}
 …
 //=============================================================================
+const bool Matcher::constant_table_absolute_addressing = true;
+const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
+void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
+  // Empty encoding
+}
+uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
+  return 0;
+}
+#ifndef PRODUCT
+void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
+  st->print("# MachConstantBaseNode (empty encoding)");
+}
+#endif
+//=============================================================================
 #ifndef PRODUCT
 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 …
+  }
   C->set_frame_complete(cbuf.code_end() - cbuf.code_begin());
+  C->set_frame_complete(cbuf.insts_size());
 #ifdef ASSERT
 …
     // testl %rax, off(%rip) // Opcode + ModRM + Disp32 == 6 bytes
     // XXX reg_mem doesn't support RIP-relative addressing yet
     cbuf.set_inst_mark();
     cbuf.relocate(cbuf.inst_mark(), relocInfo::poll_return_type, 0); // XXX
+    cbuf.set_insts_mark();
+    cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_return_type, 0); // XXX
     emit_opcode(cbuf, 0x85); // testl
     emit_rm(cbuf, 0x0, RAX_enc, 0x5); // 00 rax 101 == 0x5
     // cbuf.inst_mark() is beginning of instruction
+    // cbuf.insts_mark() is beginning of instruction
     emit_d32_reloc(cbuf, os::get_polling_page());
 //                    relocInfo::poll_return_type,
 …
   // jmp -5 # to self
   address mark = cbuf.inst_mark();  // get mark within main instrs section
   // Note that the code buffer's inst_mark is always relative to insts.
+  address mark = cbuf.insts_mark();  // get mark within main instrs section
+  // Note that the code buffer's insts_mark is always relative to insts.
   // That's why we must use the macroassembler to generate a stub.
   MacroAssembler _masm(&cbuf);
 …
   __ jump(RuntimeAddress(__ pc()));
   // Update current stubs pointer and restore code_end.
+  // Update current stubs pointer and restore insts_end.
   __ end_a_stub();
+}
 …
+{
   MacroAssembler masm(&cbuf);
   uint code_size = cbuf.code_size();
+  uint insts_size = cbuf.insts_size();
   if (UseCompressedOops) {
     masm.load_klass(rscratch1, j_rarg0);
 …
   /* WARNING these NOPs are critical so that verified entry point is properly
 bytes aligned for patching by NativeJump::patch_verified_entry() */
   int nops_cnt = 4 - ((cbuf.code_size() - code_size) & 0x3);
+  int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
   if (OptoBreakpoint) {
     // Leave space for int3
 …
+{
   // Note that the code buffer's inst_mark is always relative to insts.
+  // Note that the code buffer's insts_mark is always relative to insts.
   // That's why we must use the macroassembler to generate a handler.
   MacroAssembler _masm(&cbuf);
 …
   if (base == NULL)  return 0;  // CodeBuffer::expand failed
   int offset = __ offset();
   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->instructions_begin()));
+  __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
   __ end_a_stub();
 …
+{
   // Note that the code buffer's inst_mark is always relative to insts.
+  // Note that the code buffer's insts_mark is always relative to insts.
   // That's why we must use the macroassembler to generate a handler.
   MacroAssembler _masm(&cbuf);
 …
   __ end_a_stub();
   return offset;
+}
-static void emit_double_constant(CodeBuffer& cbuf, double x) {
-  int mark = cbuf.insts()->mark_off();
-  MacroAssembler _masm(&cbuf);
-  address double_address = __ double_constant(x);
-  cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
-  emit_d32_reloc(cbuf,
-                 (int) (double_address - cbuf.code_end() - 4),
-                 internal_word_Relocation::spec(double_address),
-                 RELOC_DISP32);
+}
-static void emit_float_constant(CodeBuffer& cbuf, float x) {
-  int mark = cbuf.insts()->mark_off();
-  MacroAssembler _masm(&cbuf);
-  address float_address = __ float_constant(x);
-  cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
-  emit_d32_reloc(cbuf,
-                 (int) (float_address - cbuf.code_end() - 4),
-                 internal_word_Relocation::spec(float_address),
-                 RELOC_DISP32);
+}
 …
+{
   return can_be_java_arg(reg);
+}
+bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
+  // In 64 bit mode a code which use multiply when
+  // devisor is constant is faster than hardware
+  // DIV instruction (it uses MulHiL).
+  return false;
+}
 …
     // JMP, CALL
     Label* l = $labl$$label;
     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0);
+    emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0);
   %}
 …
     // JMP, CALL
     Label* l = $labl$$label;
     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
     emit_d8(cbuf, disp);
 …
     $$$emit8$primary;
     emit_cc(cbuf, $secondary, $cop$$cmpcode);
     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0);
+    emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0);
   %}
 …
     Label *l = $labl$$label;
     emit_cc(cbuf, $primary, $cop$$cmpcode);
     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
     emit_d8(cbuf, disp);
 …
     // CALL Java_To_Interpreter
     // This is the instruction starting address for relocation info.
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     // CALL directly to the runtime
     emit_d32_reloc(cbuf,
                    (int) ($meth$$method - ((intptr_t) cbuf.code_end()) - 4),
+                   (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
                    runtime_call_Relocation::spec(),
                    RELOC_DISP32);
 …
   enc_class preserve_SP %{
     debug_only(int off0 = cbuf.code_size());
+    debug_only(int off0 = cbuf.insts_size());
     MacroAssembler _masm(&cbuf);
     // RBP is preserved across all calls, even compiled calls.
     // Use it to preserve RSP in places where the callee might change the SP.
     __ movptr(rbp_mh_SP_save, rsp);
     debug_only(int off1 = cbuf.code_size());
+    debug_only(int off1 = cbuf.insts_size());
     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
   %}
 …
     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
     // determine who we intended to call.
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     if (!_method) {
       emit_d32_reloc(cbuf,
                      (int) ($meth$$method - ((intptr_t) cbuf.code_end()) - 4),
+                     (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
                      runtime_call_Relocation::spec(),
                      RELOC_DISP32);
     } else if (_optimized_virtual) {
       emit_d32_reloc(cbuf,
                      (int) ($meth$$method - ((intptr_t) cbuf.code_end()) - 4),
+                     (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
                      opt_virtual_call_Relocation::spec(),
                      RELOC_DISP32);
     } else {
       emit_d32_reloc(cbuf,
                      (int) ($meth$$method - ((intptr_t) cbuf.code_end()) - 4),
+                     (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
                      static_call_Relocation::spec(),
                      RELOC_DISP32);
 …
     // Generate  "movq rax, -1", placeholder instruction to load oop-info
     // emit_call_dynamic_prologue( cbuf );
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     // movq rax, -1
 …
                    (int64_t) Universe::non_oop_word(),
                    oop_Relocation::spec_for_immediate(), RELOC_IMM64);
     address virtual_call_oop_addr = cbuf.inst_mark();
+    address virtual_call_oop_addr = cbuf.insts_mark();
     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
     // who we intended to call.
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     emit_d32_reloc(cbuf,
                    (int) ($meth$$method - ((intptr_t) cbuf.code_end()) - 4),
+                   (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
                    virtual_call_Relocation::spec(virtual_call_oop_addr),
                    RELOC_DISP32);
 …
     // callq *disp(%rax)
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     $$$emit8$primary;
     if (disp < 0x80) {
 …
       emit_d64(cbuf, $src$$constant);
+    }
-  %}
-  enc_class load_immF(regF dst, immF con)
-  %{
-    // XXX reg_mem doesn't support RIP-relative addressing yet
-    emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
-    emit_float_constant(cbuf, $con$$constant);
-  %}
-  enc_class load_immD(regD dst, immD con)
-  %{
-    // XXX reg_mem doesn't support RIP-relative addressing yet
-    emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
-    emit_double_constant(cbuf, $con$$constant);
-  %}
-  enc_class load_conF (regF dst, immF con) %{    // Load float constant
-    emit_opcode(cbuf, 0xF3);
-    if ($dst$$reg >= 8) {
-      emit_opcode(cbuf, Assembler::REX_R);
+    }
-    emit_opcode(cbuf, 0x0F);
-    emit_opcode(cbuf, 0x10);
-    emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
-    emit_float_constant(cbuf, $con$$constant);
-  %}
-  enc_class load_conD (regD dst, immD con) %{    // Load double constant
-    // UseXmmLoadAndClearUpper ? movsd(dst, con) : movlpd(dst, con)
-    emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
-    if ($dst$$reg >= 8) {
-      emit_opcode(cbuf, Assembler::REX_R);
+    }
-    emit_opcode(cbuf, 0x0F);
-    emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
-    emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
-    emit_double_constant(cbuf, $con$$constant);
   %}
 …
     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
     emit_d32(cbuf, 0x00);
-  %}
-  enc_class jump_enc(rRegL switch_val, rRegI dest) %{
-    MacroAssembler masm(&cbuf);
-    Register switch_reg = as_Register($switch_val$$reg);
-    Register dest_reg   = as_Register($dest$$reg);
-    address table_base  = masm.address_table_constant(_index2label);
-    // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
-    // to do that and the compiler is using that register as one it can allocate.
-    // So we build it all by hand.
-    // Address index(noreg, switch_reg, Address::times_1);
-    // ArrayAddress dispatch(table, index);
-    Address dispatch(dest_reg, switch_reg, Address::times_1);
-    masm.lea(dest_reg, InternalAddress(table_base));
-    masm.jmp(dispatch);
-  %}
-  enc_class jump_enc_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
-    MacroAssembler masm(&cbuf);
-    Register switch_reg = as_Register($switch_val$$reg);
-    Register dest_reg   = as_Register($dest$$reg);
-    address table_base  = masm.address_table_constant(_index2label);
-    // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
-    // to do that and the compiler is using that register as one it can allocate.
-    // So we build it all by hand.
-    // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant, (int)$offset$$constant);
-    // ArrayAddress dispatch(table, index);
-    Address dispatch(dest_reg, switch_reg, (Address::ScaleFactor)$shift$$constant, (int)$offset$$constant);
-    masm.lea(dest_reg, InternalAddress(table_base));
-    masm.jmp(dispatch);
-  %}
-  enc_class jump_enc_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
-    MacroAssembler masm(&cbuf);
-    Register switch_reg = as_Register($switch_val$$reg);
-    Register dest_reg   = as_Register($dest$$reg);
-    address table_base  = masm.address_table_constant(_index2label);
-    // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
-    // to do that and the compiler is using that register as one it can allocate.
-    // So we build it all by hand.
-    // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
-    // ArrayAddress dispatch(table, index);
-    Address dispatch(dest_reg, switch_reg, (Address::ScaleFactor)$shift$$constant);
-    masm.lea(dest_reg, InternalAddress(table_base));
-    masm.jmp(dispatch);
   %}
 …
   enc_class enc_rethrow()
   %{
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xE9); // jmp entry
     emit_d32_reloc(cbuf,
                    (int) (OptoRuntime::rethrow_stub() - cbuf.code_end() - 4),
+                   (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
                    runtime_call_Relocation::spec(),
                    RELOC_DISP32);
 …
     address signmask_address = (address) StubRoutines::x86::float_sign_mask();
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     if (dstenc >= 8) {
       emit_opcode(cbuf, Assembler::REX_R);
 …
     address signmask_address = (address) StubRoutines::x86::double_sign_mask();
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0x66);
     if (dstenc >= 8) {
 …
     address signflip_address = (address) StubRoutines::x86::float_sign_flip();
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     if (dstenc >= 8) {
       emit_opcode(cbuf, Assembler::REX_R);
 …
     address signflip_address = (address) StubRoutines::x86::double_sign_flip();
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0x66);
     if (dstenc >= 8) {
 …
     // call f2i_fixup
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xE8);
     emit_d32_reloc(cbuf,
                    (int)
                    (StubRoutines::x86::f2i_fixup() - cbuf.code_end() - 4),
+                   (StubRoutines::x86::f2i_fixup() - cbuf.insts_end() - 4),
                    runtime_call_Relocation::spec(),
                    RELOC_DISP32);
 …
     // cmpq $dst, [0x8000000000000000]
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
     emit_opcode(cbuf, 0x39);
 …
     // call f2l_fixup
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xE8);
     emit_d32_reloc(cbuf,
                    (int)
                    (StubRoutines::x86::f2l_fixup() - cbuf.code_end() - 4),
+                   (StubRoutines::x86::f2l_fixup() - cbuf.insts_end() - 4),
                    runtime_call_Relocation::spec(),
                    RELOC_DISP32);
 …
     // call d2i_fixup
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xE8);
     emit_d32_reloc(cbuf,
                    (int)
                    (StubRoutines::x86::d2i_fixup() - cbuf.code_end() - 4),
+                   (StubRoutines::x86::d2i_fixup() - cbuf.insts_end() - 4),
                    runtime_call_Relocation::spec(),
                    RELOC_DISP32);
 …
     // cmpq $dst, [0x8000000000000000]
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
     emit_opcode(cbuf, 0x39);
 …
     // call d2l_fixup
     cbuf.set_inst_mark();
+    cbuf.set_insts_mark();
     emit_opcode(cbuf, 0xE8);
     emit_d32_reloc(cbuf,
                    (int)
                    (StubRoutines::x86::d2l_fixup() - cbuf.code_end() - 4),
+                   (StubRoutines::x86::d2l_fixup() - cbuf.insts_end() - 4),
                    runtime_call_Relocation::spec(),
                    RELOC_DISP32);
 …
     // testl %rax, off(%rip) // Opcode + ModRM + Disp32 == 6 bytes
     // XXX reg_mem doesn't support RIP-relative addressing yet
     cbuf.set_inst_mark();
     cbuf.relocate(cbuf.inst_mark(), relocInfo::poll_type, 0); // XXX
+    cbuf.set_insts_mark();
+    cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_type, 0); // XXX
     emit_opcode(cbuf, 0x85); // testl
     emit_rm(cbuf, 0x0, RAX_enc, 0x5); // 00 rax 101 == 0x5
     // cbuf.inst_mark() is beginning of instruction
+    // cbuf.insts_mark() is beginning of instruction
     emit_d32_reloc(cbuf, os::get_polling_page());
 //                    relocInfo::poll_type,
 …
 %}
+instruct loadConP(rRegP dst, immP src)
+%{
+  match(Set dst src);
+  format %{ "movq    $dst, $src\t# ptr" %}
+  ins_encode(load_immP(dst, src));
+instruct loadConP(rRegP dst, immP con) %{
+  match(Set dst con);
+  format %{ "movq    $dst, $con\t# ptr" %}
+  ins_encode(load_immP(dst, con));
   ins_pipe(ialu_reg_fat); // XXX
 %}
 …
 %}
+instruct loadConF(regF dst, immF src)
+%{
+  match(Set dst src);
+instruct loadConF(regF dst, immF con) %{
+  match(Set dst con);
   ins_cost(125);
+  format %{ "movss   $dst, [$src]" %}
+  ins_encode(load_conF(dst, src));
+  format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ movflt($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 // Use the same format since predicate() can not be used here.
+instruct loadConD(regD dst, immD src)
+%{
+  match(Set dst src);
+instruct loadConD(regD dst, immD con) %{
+  match(Set dst con);
   ins_cost(125);
+  format %{ "movsd   $dst, [$src]" %}
+  ins_encode(load_conD(dst, src));
+  format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ movdbl($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
   ins_pipe( ialu_reg );
 %}
-instruct loadI_reversed(rRegI dst, memory src) %{
-  match(Set dst (ReverseBytesI (LoadI src)));
-  format %{ "bswap_movl $dst, $src" %}
-  opcode(0x8B, 0x0F, 0xC8); /* Opcode 8B 0F C8 */
-  ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src), REX_reg(dst), OpcS, opc3_reg(dst));
-  ins_pipe( ialu_reg_mem );
-%}
-instruct loadL_reversed(rRegL dst, memory src) %{
-  match(Set dst (ReverseBytesL (LoadL src)));
-  format %{ "bswap_movq $dst, $src" %}
-  opcode(0x8B, 0x0F, 0xC8); /* Opcode 8B 0F C8 */
-  ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src), REX_reg_wide(dst), OpcS, opc3_reg(dst));
-  ins_pipe( ialu_reg_mem );
-%}
-instruct storeI_reversed(memory dst, rRegI src) %{
-  match(Set dst (StoreI dst (ReverseBytesI  src)));
-  format %{ "movl_bswap $dst, $src" %}
-  opcode(0x0F, 0xC8, 0x89); /* Opcode 0F C8 89 */
-  ins_encode( REX_reg(src), OpcP, opc2_reg(src), REX_reg_mem(src, dst), OpcT, reg_mem(src, dst) );
-  ins_pipe( ialu_mem_reg );
-%}
-instruct storeL_reversed(memory dst, rRegL src) %{
-  match(Set dst (StoreL dst (ReverseBytesL  src)));
-  format %{ "movq_bswap $dst, $src" %}
-  opcode(0x0F, 0xC8, 0x89); /* Opcode 0F C8 89 */
-  ins_encode( REX_reg_wide(src), OpcP, opc2_reg(src), REX_reg_mem_wide(src, dst), OpcT, reg_mem(src, dst) );
-  ins_pipe( ialu_mem_reg );
-%}
 //---------- Zeros Count Instructions ------------------------------------------
 …
   effect(TEMP dest);
   format %{ "leaq    $dest, table_base\n\t"
+  format %{ "leaq    $dest, [$constantaddress]\n\t"
             "jmp     [$dest + $switch_val << $shift]\n\t" %}
+  ins_encode(jump_enc_offset(switch_val, shift, dest));
+  ins_encode %{
+    // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
+    // to do that and the compiler is using that register as one it can allocate.
+    // So we build it all by hand.
+    // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
+    // ArrayAddress dispatch(table, index);
+    Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
+    __ lea($dest$$Register, $constantaddress);
+    __ jmp(dispatch);
+  %}
   ins_pipe(pipe_jmp);
   ins_pc_relative(1);
 …
   effect(TEMP dest);
   format %{ "leaq    $dest, table_base\n\t"
+  format %{ "leaq    $dest, [$constantaddress]\n\t"
             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
+  ins_encode(jump_enc_addr(switch_val, shift, offset, dest));
+  ins_encode %{
+    // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
+    // to do that and the compiler is using that register as one it can allocate.
+    // So we build it all by hand.
+    // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
+    // ArrayAddress dispatch(table, index);
+    Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
+    __ lea($dest$$Register, $constantaddress);
+    __ jmp(dispatch);
+  %}
   ins_pipe(pipe_jmp);
   ins_pc_relative(1);
 …
   effect(TEMP dest);
   format %{ "leaq    $dest, table_base\n\t"
+  format %{ "leaq    $dest, [$constantaddress]\n\t"
             "jmp     [$dest + $switch_val]\n\t" %}
+  ins_encode(jump_enc(switch_val, dest));
+  ins_encode %{
+    // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
+    // to do that and the compiler is using that register as one it can allocate.
+    // So we build it all by hand.
+    // Address index(noreg, switch_reg, Address::times_1);
+    // ArrayAddress dispatch(table, index);
+    Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
+    __ lea($dest$$Register, $constantaddress);
+    __ jmp(dispatch);
+  %}
   ins_pipe(pipe_jmp);
   ins_pc_relative(1);
 …
 %}
+instruct cmpF_cc_imm(rFlagsRegU cr, regF src1, immF src2)
+%{
+  match(Set cr (CmpF src1 src2));
+instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
+  match(Set cr (CmpF src con));
   ins_cost(145);
   format %{ "ucomiss $src1, $src2\n\t"
+  format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
             "jnp,s   exit\n\t"
             "pushfq\t# saw NaN, set CF\n\t"
 …
             "popfq\n"
     "exit:   nop\t# avoid branch to branch" %}
+  opcode(0x0F, 0x2E);
+  ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, load_immF(src1, src2),
+             cmpfp_fixup);
+  ins_encode %{
+    Label L_exit;
+    __ ucomiss($src$$XMMRegister, $constantaddress($con));
+    __ jcc(Assembler::noParity, L_exit);
+    __ pushf();
+    __ andq(rsp, 0xffffff2b);
+    __ popf();
+    __ bind(L_exit);
+    __ nop();
+  %}
   ins_pipe(pipe_slow);
 %}
+instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src1, immF src2) %{
+  match(Set cr (CmpF src1 src2));
+instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
+  match(Set cr (CmpF src con));
   ins_cost(100);
+  format %{ "ucomiss $src1, $src2" %}
+  opcode(0x0F, 0x2E);
+  ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, load_immF(src1, src2));
+  format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
+  ins_encode %{
+    __ ucomiss($src$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct cmpD_cc_imm(rFlagsRegU cr, regD src1, immD src2)
+%{
+  match(Set cr (CmpD src1 src2));
+instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
+  match(Set cr (CmpD src con));
   ins_cost(145);
   format %{ "ucomisd $src1, [$src2]\n\t"
+  format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
             "jnp,s   exit\n\t"
             "pushfq\t# saw NaN, set CF\n\t"
 …
             "popfq\n"
     "exit:   nop\t# avoid branch to branch" %}
+  opcode(0x66, 0x0F, 0x2E);
+  ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, load_immD(src1, src2),
+             cmpfp_fixup);
+  ins_encode %{
+    Label L_exit;
+    __ ucomisd($src$$XMMRegister, $constantaddress($con));
+    __ jcc(Assembler::noParity, L_exit);
+    __ pushf();
+    __ andq(rsp, 0xffffff2b);
+    __ popf();
+    __ bind(L_exit);
+    __ nop();
+  %}
   ins_pipe(pipe_slow);
 %}
+instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src1, immD src2) %{
+  match(Set cr (CmpD src1 src2));
+instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
+  match(Set cr (CmpD src con));
   ins_cost(100);
+  format %{ "ucomisd $src1, [$src2]" %}
+  opcode(0x66, 0x0F, 0x2E);
+  ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, load_immD(src1, src2));
+  format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
+  ins_encode %{
+    __ ucomisd($src$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 // Compare into -1,0,1
+instruct cmpF_imm(rRegI dst, regF src1, immF src2, rFlagsReg cr)
+%{
+  match(Set dst (CmpF3 src1 src2));
+instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
+  match(Set dst (CmpF3 src con));
   effect(KILL cr);
   ins_cost(275);
   format %{ "ucomiss $src1, [$src2]\n\t"
+  format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
             "movl    $dst, #-1\n\t"
             "jp,s    done\n\t"
 …
             "movzbl  $dst, $dst\n"
     "done:" %}
+  opcode(0x0F, 0x2E);
+  ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, load_immF(src1, src2),
+             cmpfp3(dst));
+  ins_encode %{
+    Label L_done;
+    Register Rdst = $dst$$Register;
+    __ ucomiss($src$$XMMRegister, $constantaddress($con));
+    __ movl(Rdst, -1);
+    __ jcc(Assembler::parity, L_done);
+    __ jcc(Assembler::below, L_done);
+    __ setb(Assembler::notEqual, Rdst);
+    __ movzbl(Rdst, Rdst);
+    __ bind(L_done);
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 // Compare into -1,0,1
+instruct cmpD_imm(rRegI dst, regD src1, immD src2, rFlagsReg cr)
+%{
+  match(Set dst (CmpD3 src1 src2));
+instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
+  match(Set dst (CmpD3 src con));
   effect(KILL cr);
   ins_cost(275);
   format %{ "ucomisd $src1, [$src2]\n\t"
+  format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
             "movl    $dst, #-1\n\t"
             "jp,s    done\n\t"
 …
             "movzbl  $dst, $dst\n"
     "done:" %}
+  opcode(0x66, 0x0F, 0x2E);
+  ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, load_immD(src1, src2),
+             cmpfp3(dst));
+  ins_encode %{
+    Register Rdst = $dst$$Register;
+    Label L_done;
+    __ ucomisd($src$$XMMRegister, $constantaddress($con));
+    __ movl(Rdst, -1);
+    __ jcc(Assembler::parity, L_done);
+    __ jcc(Assembler::below, L_done);
+    __ setb(Assembler::notEqual, Rdst);
+    __ movzbl(Rdst, Rdst);
+    __ bind(L_done);
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct addF_imm(regF dst, immF src)
+%{
+  match(Set dst (AddF dst src));
+  format %{ "addss   $dst, [$src]" %}
+instruct addF_imm(regF dst, immF con) %{
+  match(Set dst (AddF dst con));
+  format %{ "addss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF3, 0x0F, 0x58);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
+  ins_encode %{
+    __ addss($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct addD_imm(regD dst, immD src)
+%{
+  match(Set dst (AddD dst src));
+  format %{ "addsd   $dst, [$src]" %}
+instruct addD_imm(regD dst, immD con) %{
+  match(Set dst (AddD dst con));
+  format %{ "addsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF2, 0x0F, 0x58);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
+  ins_encode %{
+    __ addsd($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct subF_imm(regF dst, immF src)
+%{
+  match(Set dst (SubF dst src));
+  format %{ "subss   $dst, [$src]" %}
+instruct subF_imm(regF dst, immF con) %{
+  match(Set dst (SubF dst con));
+  format %{ "subss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF3, 0x0F, 0x5C);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
+  ins_encode %{
+    __ subss($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct subD_imm(regD dst, immD src)
+%{
+  match(Set dst (SubD dst src));
+  format %{ "subsd   $dst, [$src]" %}
+instruct subD_imm(regD dst, immD con) %{
+  match(Set dst (SubD dst con));
+  format %{ "subsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF2, 0x0F, 0x5C);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
+  ins_encode %{
+    __ subsd($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct mulF_imm(regF dst, immF src)
+%{
+  match(Set dst (MulF dst src));
+  format %{ "mulss   $dst, [$src]" %}
+instruct mulF_imm(regF dst, immF con) %{
+  match(Set dst (MulF dst con));
+  format %{ "mulss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF3, 0x0F, 0x59);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
+  ins_encode %{
+    __ mulss($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct mulD_imm(regD dst, immD src)
+%{
+  match(Set dst (MulD dst src));
+  format %{ "mulsd   $dst, [$src]" %}
+instruct mulD_imm(regD dst, immD con) %{
+  match(Set dst (MulD dst con));
+  format %{ "mulsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF2, 0x0F, 0x59);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
+  ins_encode %{
+    __ mulsd($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct divF_imm(regF dst, immF src)
+%{
+  match(Set dst (DivF dst src));
+  format %{ "divss   $dst, [$src]" %}
+instruct divF_imm(regF dst, immF con) %{
+  match(Set dst (DivF dst con));
+  format %{ "divss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF3, 0x0F, 0x5E);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
+  ins_encode %{
+    __ divss($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct divD_imm(regD dst, immD src)
+%{
+  match(Set dst (DivD dst src));
+  format %{ "divsd   $dst, [$src]" %}
+instruct divD_imm(regD dst, immD con) %{
+  match(Set dst (DivD dst con));
+  format %{ "divsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF2, 0x0F, 0x5E);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
+  ins_encode %{
+    __ divsd($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct sqrtF_imm(regF dst, immF src)
+%{
+  match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
+  format %{ "sqrtss  $dst, [$src]" %}
+instruct sqrtF_imm(regF dst, immF con) %{
+  match(Set dst (ConvD2F (SqrtD (ConvF2D con))));
+  format %{ "sqrtss  $dst, [$constantaddress]\t# load from constant table: float=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF3, 0x0F, 0x51);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
+  ins_encode %{
+    __ sqrtss($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
+instruct sqrtD_imm(regD dst, immD src)
+%{
+  match(Set dst (SqrtD src));
+  format %{ "sqrtsd  $dst, [$src]" %}
+instruct sqrtD_imm(regD dst, immD con) %{
+  match(Set dst (SqrtD con));
+  format %{ "sqrtsd  $dst, [$constantaddress]\t# load from constant table: double=$con" %}
   ins_cost(150); // XXX
+  opcode(0xF2, 0x0F, 0x51);
+  ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
+  ins_encode %{
+    __ sqrtsd($dst$$XMMRegister, $constantaddress($con));
+  %}
   ins_pipe(pipe_slow);
 %}
 …
 %}
 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rbx_RegI cnt2,
                         rax_RegI result, regD tmp1, regD tmp2, rFlagsReg cr)
+instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
+                        rax_RegI result, regD tmp1, rFlagsReg cr)
 %{
   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
+  effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
+  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
   ins_encode %{
     __ string_compare($str1$$Register, $str2$$Register,
                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
                       $tmp1$$XMMRegister, $tmp2$$XMMRegister);
+                      $tmp1$$XMMRegister);
   %}
   ins_pipe( pipe_slow );
 …
     if ($cop$$cmpcode == Assembler::notEqual) {
        // the two jumps 6 bytes apart so the jump distances are too
        parity_disp = l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0;
+       parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
     } else if ($cop$$cmpcode == Assembler::equal) {
        parity_disp = 6;
 …
     $$$emit8$primary;
     emit_cc(cbuf, $secondary, $cop$$cmpcode);
     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
     emit_d32(cbuf, disp);
   %}
 …
     int parity_disp = -1;
     if ($cop$$cmpcode == Assembler::notEqual) {
       parity_disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
+      parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
     } else if ($cop$$cmpcode == Assembler::equal) {
       parity_disp = 2;
 …
     emit_d8(cbuf, parity_disp);
     emit_cc(cbuf, $primary, $cop$$cmpcode);
     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
+    int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
     emit_d8(cbuf, disp);
     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");

/branches/vendor/oracle/openjdk6/b24 (added)	merged: 308
/branches/vendor/oracle/openjdk6/current	merged: 307

Context Navigation

Legend:

Download in other formats: