Deoptimize to VM when hooks are enabled

* libguile/vm.c (vm_clear_mcode_return_addresses): New helper.
(vm_recompute_disable_mcode): Force a thread to deoptimize if hooks
become enabled.
(scm_call_n): Don't enter mcode if it's disabled.  Also check the right
flag for when to run the abort hook (the abort_hook_enabled flag).
* libguile/vm-engine.c (instrument-entry, instrument-loop)
(return-values, abort, compose-continuation): Don't enter mcode if mcode
is disabled for this thread.

libguile/vm-engine.c

@@ -458,35 +458,38 @@ VM_NAME (scm_thread *thread)
    */
   VM_DEFINE_OP (5, instrument_entry, "instrument-entry", OP2 (X32, N32))
     {
-      int32_t data_offset = ip[1];
+      if (!VP->disable_mcode)
-      struct scm_jit_function_data *data;
-
-      data = (struct scm_jit_function_data *) (ip + data_offset);
-
-      if (data->mcode)
         {
-          SYNC_IP ();
+          struct scm_jit_function_data *data;
-          scm_jit_enter_mcode (thread, data->mcode);
-          CACHE_REGISTER ();
-          NEXT (0);
-        }
 
-      if (data->counter >= scm_jit_counter_threshold)
+          int32_t data_offset = ip[1];
-        {
+          data = (struct scm_jit_function_data *) (ip + data_offset);
-          const uint8_t *mcode;
 
-          SYNC_IP ();
+          if (data->mcode)
-          mcode = scm_jit_compute_mcode (thread, data);
-
-          if (mcode)
             {
-              scm_jit_enter_mcode (thread, mcode);
+              SYNC_IP ();
+              scm_jit_enter_mcode (thread, data->mcode);
               CACHE_REGISTER ();
               NEXT (0);
             }
+
+          if (data->counter >= scm_jit_counter_threshold)
+            {
+              const uint8_t *mcode;
+
+              SYNC_IP ();
+              mcode = scm_jit_compute_mcode (thread, data);
+
+              if (mcode)
+                {
+                  scm_jit_enter_mcode (thread, mcode);
+                  CACHE_REGISTER ();
+                  NEXT (0);
+                }
+            }
+          else
+            data->counter += SCM_JIT_COUNTER_ENTRY_INCREMENT;
         }
-      else
-        data->counter += SCM_JIT_COUNTER_ENTRY_INCREMENT;
 
       APPLY_HOOK ();
 
@@ -572,18 +575,19 @@ VM_NAME (scm_thread *thread)
       old_fp = VP->fp;
       VP->fp = SCM_FRAME_DYNAMIC_LINK (old_fp);
 
-      mcode = SCM_FRAME_MACHINE_RETURN_ADDRESS (old_fp);
+      if (!VP->disable_mcode)
-      if (mcode)
         {
-          scm_jit_enter_mcode (thread, mcode);
+          mcode = SCM_FRAME_MACHINE_RETURN_ADDRESS (old_fp);
-          CACHE_REGISTER ();
+          if (mcode)
-          NEXT (0);
+            {
-        }
+              scm_jit_enter_mcode (thread, mcode);
-      else
+              CACHE_REGISTER ();
-        {
+              NEXT (0);
-          ip = SCM_FRAME_VIRTUAL_RETURN_ADDRESS (old_fp);
+            }
-          NEXT (0);
         }
+
+      ip = SCM_FRAME_VIRTUAL_RETURN_ADDRESS (old_fp);
+      NEXT (0);
     }
 
 
@@ -695,7 +699,7 @@ VM_NAME (scm_thread *thread)
       SYNC_IP ();
       mcode = CALL_INTRINSIC (compose_continuation, (thread, vmcont));
 
-      if (mcode)
+      if (mcode && !VP->disable_mcode)
         {
           scm_jit_enter_mcode (thread, mcode);
           CACHE_REGISTER ();
@@ -717,27 +721,30 @@ VM_NAME (scm_thread *thread)
    */
   VM_DEFINE_OP (14, instrument_loop, "instrument-loop", OP2 (X32, N32))
     {
-      int32_t data_offset = ip[1];
+      if (!VP->disable_mcode)
-      struct scm_jit_function_data *data;
-
-      data = (struct scm_jit_function_data *) (ip + data_offset);
-
-      if (data->counter >= scm_jit_counter_threshold)
         {
-          const uint8_t *mcode;
+          int32_t data_offset = ip[1];
+          struct scm_jit_function_data *data;
 
-          SYNC_IP ();
+          data = (struct scm_jit_function_data *) (ip + data_offset);
-          mcode = scm_jit_compute_mcode (thread, data);
 
-          if (mcode)
+          if (data->counter >= scm_jit_counter_threshold)
             {
-              scm_jit_enter_mcode (thread, mcode);
+              const uint8_t *mcode;
-              CACHE_REGISTER ();
+
-              NEXT (0);
+              SYNC_IP ();
+              mcode = scm_jit_compute_mcode (thread, data);
+
+              if (mcode)
+                {
+                  scm_jit_enter_mcode (thread, mcode);
+                  CACHE_REGISTER ();
+                  NEXT (0);
+                }
             }
+          else
+            data->counter += SCM_JIT_COUNTER_LOOP_INCREMENT;
         }
-      else
-        data->counter += SCM_JIT_COUNTER_LOOP_INCREMENT;
 
       NEXT (2);
     }
@@ -782,7 +789,7 @@ VM_NAME (scm_thread *thread)
 
       ABORT_HOOK ();
 
-      if (mcode)
+      if (mcode && !VP->disable_mcode)
         scm_jit_enter_mcode (thread, mcode);
 
       CACHE_REGISTER ();

libguile/vm.c

@@ -198,6 +198,18 @@ scm_i_capture_current_stack (void)
                         0);
 }
 
+/* Call to force a thread to go back to the interpreter, for example
+   when single-stepping is enabled.  */
+static void
+vm_clear_mcode_return_addresses (scm_thread *thread)
+{
+  union scm_vm_stack_element *fp;
+  struct scm_vm *vp = &thread->vm;
+
+  for (fp = vp->fp; fp < vp->stack_top; fp = SCM_FRAME_DYNAMIC_LINK (fp))
+    SCM_FRAME_SET_MACHINE_RETURN_ADDRESS (fp, NULL);
+}
+
 #define FOR_EACH_HOOK(M) \
   M(apply) \
   M(return) \
@@ -219,12 +231,17 @@ vm_hook_compute_enabled (scm_thread *thread, SCM hook, uint8_t *enabled)
 static void
 vm_recompute_disable_mcode (scm_thread *thread)
 {
+  uint8_t was_disabled = thread->vm.disable_mcode;
   thread->vm.disable_mcode = 0;
+
 #define DISABLE_MCODE_IF_HOOK_ENABLED(h) \
   if (thread->vm.h##_hook_enabled)       \
     thread->vm.disable_mcode = 1;
-  FOR_EACH_HOOK (DISABLE_MCODE_IF_HOOK_ENABLED)
+  FOR_EACH_HOOK (DISABLE_MCODE_IF_HOOK_ENABLED);
 #undef DISABLE_MCODE_IF_HOOK_ENABLED
+
+  if (thread->vm.disable_mcode && !was_disabled)
+    vm_clear_mcode_return_addresses (thread);
 }
 
 static int
@@ -1499,9 +1516,9 @@ scm_call_n (SCM proc, SCM *argv, size_t nargs)
         uint8_t *mcode = vp->mra_after_abort;
         scm_gc_after_nonlocal_exit ();
         /* Non-local return.  */
-        if (vp->trace_level)
+        if (vp->abort_hook_enabled)
           invoke_abort_hook (thread);
-        if (mcode)
+        if (mcode && !vp->disable_mcode)
           scm_jit_enter_mcode (thread, mcode);
       }
     else