quick check

memory_profiler_script.py

@@ -22,6 +22,7 @@ class MemoryProfiler:
         self.process = psutil.Process()
         self.running = False
         self.thread = None
+        self.checkpoint_counter = 0
         
     def start_monitoring(self, interval: float = 1.0):
         """Start continuous memory monitoring"""
@@ -114,21 +115,42 @@ class MemoryProfiler:
                 
                 self.data.append(data_point)
                 
-                # Print periodic updates
+                # Print periodic updates and save partial data
                 if len(self.data) % 10 == 0:  # Every 10 samples
                     print(f"🔍 Memory: RSS={rss_gb:.2f}GB, VRAM={vram_gb:.2f}GB, Sys={sys_used_gb:.1f}GB")
                     
+                    # Save partial data every 30 samples in case of crash
+                    if len(self.data) % 30 == 0:
+                        self._save_partial_data()
+                    
             except Exception as e:
                 print(f"Monitoring error: {e}")
                 
             time.sleep(interval)
     
+    def _save_partial_data(self):
+        """Save partial data to prevent loss on crash"""
+        try:
+            partial_file = f"memory_profile_partial_{self.checkpoint_counter}.json"
+            with open(partial_file, 'w') as f:
+                json.dump({
+                    'timeline': self.data,
+                    'status': 'partial_save',
+                    'samples': len(self.data)
+                }, f, indent=2)
+            self.checkpoint_counter += 1
+        except Exception as e:
+            print(f"Failed to save partial data: {e}")
+    
     def log_checkpoint(self, checkpoint_name: str):
         """Log a specific checkpoint"""
         if self.data:
             self.data[-1]['checkpoint'] = checkpoint_name
             latest = self.data[-1]
             print(f"📍 CHECKPOINT [{checkpoint_name}]: RSS={latest['rss_gb']:.2f}GB, VRAM={latest['vram_gb']:.2f}GB")
+            
+            # Save checkpoint data immediately
+            self._save_partial_data()
 
 def run_with_profiling(config_path: str):
     """Run the VR180 matting with memory profiling"""