scott/test2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

memleak fix hopefully

Browse Source
This commit is contained in:
Scott Register
2025-07-26 12:25:55 -07:00
parent 463f881eaf
commit ccc68a3895
1 changed files with 87 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
vr180_matting/video_processor.py
View File

@@ -12,6 +12,7 @@ import subprocess
import gc import gc
import psutil import psutil
import os import os
import sys
from .config import VR180Config from .config import VR180Config
from .detector import YOLODetector from .detector import YOLODetector
@@ -131,6 +132,15 @@ class VideoProcessor:
except ImportError: except ImportError:
pass pass
# Force Linux to release memory back to OS
if sys.platform == 'linux':
try:
import ctypes
libc = ctypes.CDLL("libc.so.6")
libc.malloc_trim(0)
except Exception as e:
print(f" Warning: Could not trim memory: {e}")
# Brief pause to allow cleanup # Brief pause to allow cleanup
time.sleep(0.1) time.sleep(0.1)
@@ -565,58 +575,90 @@ class VideoProcessor:
chunk_size, overlap_frames = self.calculate_optimal_chunking() chunk_size, overlap_frames = self.calculate_optimal_chunking()
# Process video in chunks # Process video in chunks
chunk_results = [] chunk_files = [] # Store file paths instead of frame data
temp_chunk_dir = Path(tempfile.mkdtemp(prefix="vr180_chunks_"))
for start_frame in range(0, self.total_frames, chunk_size - overlap_frames): try:
end_frame = min(start_frame + chunk_size, self.total_frames) for start_frame in range(0, self.total_frames, chunk_size - overlap_frames):
frames_to_read = end_frame - start_frame end_frame = min(start_frame + chunk_size, self.total_frames)
frames_to_read = end_frame - start_frame
chunk_idx = len(chunk_results) chunk_idx = len(chunk_files)
print(f"\nProcessing chunk {chunk_idx}: frames {start_frame}-{end_frame}") print(f"\nProcessing chunk {chunk_idx}: frames {start_frame}-{end_frame}")
# Read chunk frames # Read chunk frames
frames = self.read_video_frames( frames = self.read_video_frames(
self.config.input.video_path, self.config.input.video_path,
start_frame=start_frame, start_frame=start_frame,
num_frames=frames_to_read, num_frames=frames_to_read,
scale_factor=self.config.processing.scale_factor scale_factor=self.config.processing.scale_factor
) )
# Process chunk # Process chunk
matted_frames = self.process_chunk(frames, chunk_idx) matted_frames = self.process_chunk(frames, chunk_idx)
chunk_results.append(matted_frames)
# Update statistics # Save chunk to disk immediately to free memory
self.processing_stats['chunks_processed'] += 1 chunk_path = temp_chunk_dir / f"chunk_{chunk_idx:04d}.npz"
self.processing_stats['frames_processed'] += len(frames) print(f"Saving chunk {chunk_idx} to disk...")
np.savez_compressed(str(chunk_path), frames=matted_frames)
chunk_files.append(chunk_path)
# Memory cleanup # Free the frames from memory immediately
self.memory_manager.cleanup_memory() del matted_frames
del frames
if self.memory_manager.should_emergency_cleanup(): # Update statistics
self.memory_manager.emergency_cleanup() self.processing_stats['chunks_processed'] += 1
self.processing_stats['frames_processed'] += frames_to_read
# Merge chunks if multiple # Aggressive memory cleanup after each chunk
print("\nMerging chunks...") self._aggressive_memory_cleanup(f"chunk {chunk_idx} completion")
final_frames = self.merge_overlapping_chunks(chunk_results, overlap_frames)
# Save results # Also use memory manager cleanup
print(f"Saving {len(final_frames)} processed frames...") self.memory_manager.cleanup_memory()
self.save_video(final_frames, self.config.output.path)
# Calculate final statistics if self.memory_manager.should_emergency_cleanup():
self.processing_stats['end_time'] = time.time() self.memory_manager.emergency_cleanup()
self.processing_stats['total_duration'] = self.processing_stats['end_time'] - self.processing_stats['start_time']
if self.processing_stats['total_duration'] > 0:
self.processing_stats['processing_fps'] = self.processing_stats['frames_processed'] / self.processing_stats['total_duration']
# Print processing statistics # Load and merge chunks from disk
self._print_processing_statistics() print("\nLoading and merging chunks...")
chunk_results = []
for chunk_file in chunk_files:
print(f"Loading {chunk_file.name}...")
chunk_data = np.load(str(chunk_file))
chunk_results.append(chunk_data['frames'])
chunk_data.close() # Close the file
# Print final memory report # Merge chunks
self.memory_manager.print_memory_report() final_frames = self.merge_overlapping_chunks(chunk_results, overlap_frames)
print("Video processing completed!") # Free chunk results after merging
del chunk_results
self._aggressive_memory_cleanup("after merging chunks")
# Save results
print(f"Saving {len(final_frames)} processed frames...")
self.save_video(final_frames, self.config.output.path)
# Calculate final statistics
self.processing_stats['end_time'] = time.time()
self.processing_stats['total_duration'] = self.processing_stats['end_time'] - self.processing_stats['start_time']
if self.processing_stats['total_duration'] > 0:
self.processing_stats['processing_fps'] = self.processing_stats['frames_processed'] / self.processing_stats['total_duration']
# Print processing statistics
self._print_processing_statistics()
# Print final memory report
self.memory_manager.print_memory_report()
print("Video processing completed!")
finally:
# Clean up temporary chunk files
if temp_chunk_dir.exists():
print("Cleaning up temporary chunk files...")
shutil.rmtree(temp_chunk_dir)
def _print_processing_statistics(self): def _print_processing_statistics(self):
"""Print detailed processing statistics""" """Print detailed processing statistics"""