Fix #161532: Undoing from Texture Paint into Particle Edit corrupts data

Particle Edit undo restored its mode by calling
`ED_object_particle_edit_mode_enter_ex()` directly.

When the undo step was decoded while Texture Paint was active, the
previous mode was not exited first. Since Particle Edit mode entry adds
its mode bit with `|=`, the object ended up with the invalid combined
mode `48` (`TEXTURE_PAINT | PARTICLE_EDIT`).

Exit the current mode through its normal lifecycle before restoring
Particle Edit, following the same pattern used by sculpt undo.

Files that were already saved with mode `48` crashed during editor
initialization because `object_mode_op_string()` returned `nullptr`.
Unsupported saved modes are now rejected safely, leaving the object in
Object Mode while preserving its particle and hair data so the file can
be resaved.

Pull Request: https://projects.blender.org/blender/blender/pulls/161542

source/blender/editors/object/object_modes.cc

@@ -205,6 +205,13 @@ bool mode_set_ex(bContext *C, eObjectMode mode, bool use_undo, ReportList *repor
   }
 
   const char *opstring = object_mode_op_string((mode == OB_MODE_OBJECT) ? ob->mode : mode);
+  if (opstring == nullptr) {
+    BKE_reportf(reports,
+                RPT_ERROR,
+                "Unable to set invalid object mode %d",
+                int((mode == OB_MODE_OBJECT) ? ob->mode : mode));
+    return false;
+  }
   wmOperatorType *ot = WM_operatortype_find(opstring, false);
 
   if (!use_undo) {

source/blender/editors/physics/particle_edit_undo.cc

@@ -248,6 +248,11 @@ static void particle_undosys_step_decode(
       CTX_wm_manager(C), us->scene_ref.ptr, &scene, &view_layer);
 
   Object *ob = us->object_ref.ptr;
+  if (ob->mode != OB_MODE_PARTICLE_EDIT) {
+    /* Exit the current mode before restoring Particle Edit to clean up its runtime data and avoid
+     * combining incompatible mode bits. */
+    ed::object::mode_generic_exit(bmain, depsgraph, scene, ob);
+  }
   ED_object_particle_edit_mode_enter_ex(depsgraph, scene, ob);
 
   PTCacheEdit *edit = PE_get_current(depsgraph, scene, ob);

source/blender/editors/util/ed_util.cc

@@ -196,7 +196,7 @@ void ED_editors_init(bContext *C)
     else {
       /* TODO(@ideasman42): avoid operator calls. */
       if (obact == &ob) {
-        object::mode_set(C, mode);
+        object::mode_set_ex(C, mode, true, reports);
       }
     }
   }

tests/python/CMakeLists.txt

@@ -1730,6 +1730,7 @@ if(WITH_UI_TESTS)
     test_undo.view3d_mesh_particle_edit_mode_simple
     test_undo.view3d_multi_mode_multi_window
     test_undo.view3d_multi_mode_select
+    test_undo.view3d_particle_edit_undo_from_texture_paint
     test_undo.view3d_sculpt_dyntopo_and_edit
     test_undo.view3d_sculpt_dyntopo_simple
     test_undo.view3d_sculpt_dyntopo_stroke_toggle

tests/python/ui_simulate/test_undo.py

@@ -575,6 +575,69 @@ def view3d_texture_paint_complex():
     t.assertEqual(window.view_layer.objects.active.mode, 'OBJECT')
 
 
+def view3d_particle_edit_undo_from_texture_paint():
+    import bpy
+
+    e, t, window = ui.test_window()
+    yield from _view3d_startup_area_maximized(e)
+
+    def assert_particle_edit_valid():
+        ob = bpy.context.active_object
+        t.assertEqual(ob.mode, 'PARTICLE_EDIT')
+        t.assertEqual(bpy.context.mode, 'PARTICLE')
+        psys = ob.particle_systems.active
+        t.assertIsNotNone(psys)
+        t.assertEqual(psys.settings.type, 'HAIR')
+        t.assertGreater(len(psys.particles), 0)
+        t.assertGreater(len(psys.particles[0].hair_keys), 0)
+        t.assertTrue(bpy.ops.particle.select_all.poll())
+
+    yield from ui.call_menu(e, "Add -> Mesh -> Cube")
+    ob = window.view_layer.objects.active
+
+    yield e.ctrl.tab().t()              # Texture Paint via the mode pie.
+    yield from ui.call_operator(e, "Add Texture Paint Slot")
+    yield e.ret()                       # Accept popup.
+    yield e.ctrl.tab().o()              # Object mode.
+
+    yield from ui.call_operator(e, "Add Particle System Slot")
+    psys = ob.particle_systems.active
+    psys.settings.type = 'HAIR'
+    psys.settings.count = 8
+    psys.settings.hair_length = 1.0
+    window.view_layer.update()
+    yield
+
+    # Include the programmatic setup in a memfile undo step through a normal user action.
+    yield e.r.z().text("15").ret()
+
+    yield from ui.call_operator(e, "Particle Edit Toggle")
+    assert_particle_edit_valid()
+
+    # Establish particle-edit undo steps through the same shortcuts a user invokes.
+    yield e.alt.a()                     # Deselect all.
+    yield e.a()                         # Select all.
+
+    yield e.ctrl.tab().t()              # Texture Paint via the mode pie.
+    t.assertEqual(ob.mode, 'TEXTURE_PAINT')
+
+    # This used to combine Particle Edit with Texture Paint into invalid mode value 48.
+    yield e.ctrl.z()
+    assert_particle_edit_valid()
+
+    # Exercise the mode transition in both directions, then repeat the cross-mode undo.
+    yield e.ctrl.shift.z()
+    t.assertEqual(bpy.context.active_object.mode, 'TEXTURE_PAINT')
+    yield e.ctrl.z()
+    assert_particle_edit_valid()
+
+    # Verify the reconstructed Particle Edit session still accepts user operations.
+    yield e.alt.a()
+    assert_particle_edit_valid()
+    yield e.a()
+    assert_particle_edit_valid()
+
+
 def view3d_mesh_edit_separate():
     e, t, window = ui.test_window()
     yield from _view3d_startup_area_maximized(e)