more memleak fixes
This commit is contained in:
@@ -132,6 +132,26 @@ class VideoProcessor:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# Clear OpenCV internal caches
|
||||||
|
try:
|
||||||
|
# Clear OpenCV video capture cache
|
||||||
|
cv2.setUseOptimized(False)
|
||||||
|
cv2.setUseOptimized(True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Clear CuPy caches if available
|
||||||
|
try:
|
||||||
|
import cupy as cp
|
||||||
|
cp._default_memory_pool.free_all_blocks()
|
||||||
|
cp._default_pinned_memory_pool.free_all_blocks()
|
||||||
|
cp.get_default_memory_pool().free_all_blocks()
|
||||||
|
cp.get_default_pinned_memory_pool().free_all_blocks()
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
print(f" Warning: Could not clear CuPy cache: {e}")
|
||||||
|
|
||||||
# Force Linux to release memory back to OS
|
# Force Linux to release memory back to OS
|
||||||
if sys.platform == 'linux':
|
if sys.platform == 'linux':
|
||||||
try:
|
try:
|
||||||
@@ -623,16 +643,27 @@ class VideoProcessor:
|
|||||||
# Load and merge chunks from disk
|
# Load and merge chunks from disk
|
||||||
print("\nLoading and merging chunks...")
|
print("\nLoading and merging chunks...")
|
||||||
chunk_results = []
|
chunk_results = []
|
||||||
for chunk_file in chunk_files:
|
for i, chunk_file in enumerate(chunk_files):
|
||||||
print(f"Loading {chunk_file.name}...")
|
print(f"Loading {chunk_file.name}...")
|
||||||
chunk_data = np.load(str(chunk_file))
|
chunk_data = np.load(str(chunk_file))
|
||||||
chunk_results.append(chunk_data['frames'])
|
chunk_results.append(chunk_data['frames'])
|
||||||
chunk_data.close() # Close the file
|
chunk_data.close() # Close the file
|
||||||
|
|
||||||
|
# Delete chunk file immediately after loading to free disk space
|
||||||
|
try:
|
||||||
|
chunk_file.unlink()
|
||||||
|
print(f" Deleted chunk file {chunk_file.name}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" Warning: Could not delete chunk file: {e}")
|
||||||
|
|
||||||
|
# Aggressive cleanup every few chunks to prevent accumulation
|
||||||
|
if i % 3 == 0 and i > 0:
|
||||||
|
self._aggressive_memory_cleanup(f"after loading chunk {i}")
|
||||||
|
|
||||||
# Merge chunks
|
# Merge chunks
|
||||||
final_frames = self.merge_overlapping_chunks(chunk_results, overlap_frames)
|
final_frames = self.merge_overlapping_chunks(chunk_results, overlap_frames)
|
||||||
|
|
||||||
# Free chunk results after merging
|
# Free chunk results after merging - this is critical!
|
||||||
del chunk_results
|
del chunk_results
|
||||||
self._aggressive_memory_cleanup("after merging chunks")
|
self._aggressive_memory_cleanup("after merging chunks")
|
||||||
|
|
||||||
|
|||||||
@@ -117,8 +117,14 @@ class VR180Processor(VideoProcessor):
|
|||||||
# Combine horizontally on GPU (much faster for large arrays)
|
# Combine horizontally on GPU (much faster for large arrays)
|
||||||
combined_gpu = cp.hstack([left_gpu, right_gpu])
|
combined_gpu = cp.hstack([left_gpu, right_gpu])
|
||||||
|
|
||||||
# Transfer back to CPU
|
# Transfer back to CPU and ensure we get a copy, not a view
|
||||||
return cp.asnumpy(combined_gpu)
|
combined = cp.asnumpy(combined_gpu).copy()
|
||||||
|
|
||||||
|
# Free GPU memory immediately
|
||||||
|
del left_gpu, right_gpu, combined_gpu
|
||||||
|
cp._default_memory_pool.free_all_blocks()
|
||||||
|
|
||||||
|
return combined
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# Fallback to CPU NumPy
|
# Fallback to CPU NumPy
|
||||||
@@ -128,8 +134,8 @@ class VR180Processor(VideoProcessor):
|
|||||||
left_eye = cv2.resize(left_eye, (left_eye.shape[1], target_height))
|
left_eye = cv2.resize(left_eye, (left_eye.shape[1], target_height))
|
||||||
right_eye = cv2.resize(right_eye, (right_eye.shape[1], target_height))
|
right_eye = cv2.resize(right_eye, (right_eye.shape[1], target_height))
|
||||||
|
|
||||||
# Combine horizontally
|
# Combine horizontally and ensure we get a copy, not a view
|
||||||
combined = np.hstack([left_eye, right_eye])
|
combined = np.hstack([left_eye, right_eye]).copy()
|
||||||
return combined
|
return combined
|
||||||
|
|
||||||
def process_with_disparity_mapping(self,
|
def process_with_disparity_mapping(self,
|
||||||
@@ -176,6 +182,10 @@ class VR180Processor(VideoProcessor):
|
|||||||
with self.memory_manager.memory_monitor(f"left eye chunk {chunk_idx}"):
|
with self.memory_manager.memory_monitor(f"left eye chunk {chunk_idx}"):
|
||||||
left_matted = self._process_eye_sequence(left_eye_frames, "left", chunk_idx)
|
left_matted = self._process_eye_sequence(left_eye_frames, "left", chunk_idx)
|
||||||
|
|
||||||
|
# Free left eye frames after processing (before right eye to save memory)
|
||||||
|
del left_eye_frames
|
||||||
|
self._aggressive_memory_cleanup(f"After left eye processing chunk {chunk_idx}")
|
||||||
|
|
||||||
# Process right eye with cross-validation
|
# Process right eye with cross-validation
|
||||||
print("Processing right eye with cross-validation...")
|
print("Processing right eye with cross-validation...")
|
||||||
with self.memory_manager.memory_monitor(f"right eye chunk {chunk_idx}"):
|
with self.memory_manager.memory_monitor(f"right eye chunk {chunk_idx}"):
|
||||||
@@ -183,6 +193,10 @@ class VR180Processor(VideoProcessor):
|
|||||||
right_eye_frames, left_matted, "right", chunk_idx
|
right_eye_frames, left_matted, "right", chunk_idx
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Free right eye frames after processing
|
||||||
|
del right_eye_frames
|
||||||
|
self._aggressive_memory_cleanup(f"After right eye processing chunk {chunk_idx}")
|
||||||
|
|
||||||
# Combine results back to SBS format
|
# Combine results back to SBS format
|
||||||
combined_frames = []
|
combined_frames = []
|
||||||
for left_frame, right_frame in zip(left_matted, right_matted):
|
for left_frame, right_frame in zip(left_matted, right_matted):
|
||||||
@@ -193,6 +207,11 @@ class VR180Processor(VideoProcessor):
|
|||||||
combined = {'left': left_frame, 'right': right_frame}
|
combined = {'left': left_frame, 'right': right_frame}
|
||||||
combined_frames.append(combined)
|
combined_frames.append(combined)
|
||||||
|
|
||||||
|
# Free the individual eye results after combining
|
||||||
|
del left_matted
|
||||||
|
del right_matted
|
||||||
|
self._aggressive_memory_cleanup(f"After combining frames chunk {chunk_idx}")
|
||||||
|
|
||||||
return combined_frames
|
return combined_frames
|
||||||
|
|
||||||
def _process_eye_sequence(self,
|
def _process_eye_sequence(self,
|
||||||
@@ -395,8 +414,9 @@ class VR180Processor(VideoProcessor):
|
|||||||
|
|
||||||
matted_frames.append(matted_frame)
|
matted_frames.append(matted_frame)
|
||||||
|
|
||||||
# Free reloaded frames
|
# Free reloaded frames and video segments completely
|
||||||
del reloaded_frames
|
del reloaded_frames
|
||||||
|
del video_segments # This holds processed masks from SAM2
|
||||||
self._aggressive_memory_cleanup(f"After mask application ({eye_name} eye)")
|
self._aggressive_memory_cleanup(f"After mask application ({eye_name} eye)")
|
||||||
|
|
||||||
return matted_frames
|
return matted_frames
|
||||||
@@ -438,6 +458,10 @@ class VR180Processor(VideoProcessor):
|
|||||||
left_eye_results, right_matted
|
left_eye_results, right_matted
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# CRITICAL: Free the intermediate results to prevent memory accumulation
|
||||||
|
del left_eye_results # Don't keep left eye results after validation
|
||||||
|
del right_matted # Don't keep unvalidated right results
|
||||||
|
|
||||||
return validated_results
|
return validated_results
|
||||||
|
|
||||||
def _validate_stereo_consistency(self,
|
def _validate_stereo_consistency(self,
|
||||||
@@ -521,6 +545,20 @@ class VR180Processor(VideoProcessor):
|
|||||||
del left_areas, right_areas, area_ratios, needs_correction
|
del left_areas, right_areas, area_ratios, needs_correction
|
||||||
cp._default_memory_pool.free_all_blocks()
|
cp._default_memory_pool.free_all_blocks()
|
||||||
|
|
||||||
|
# CRITICAL: Release ALL CuPy memory back to system after validation
|
||||||
|
try:
|
||||||
|
# Force release of all GPU memory pools
|
||||||
|
cp._default_memory_pool.free_all_blocks()
|
||||||
|
cp._default_pinned_memory_pool.free_all_blocks()
|
||||||
|
|
||||||
|
# Clear CuPy cache completely
|
||||||
|
cp.get_default_memory_pool().free_all_blocks()
|
||||||
|
cp.get_default_pinned_memory_pool().free_all_blocks()
|
||||||
|
|
||||||
|
print(f" CuPy memory pools cleared")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" Warning: Could not clear CuPy memory pools: {e}")
|
||||||
|
|
||||||
correction_count = sum(needs_correction_all)
|
correction_count = sum(needs_correction_all)
|
||||||
print(f" GPU validation complete: {correction_count}/{total_frames} frames need correction")
|
print(f" GPU validation complete: {correction_count}/{total_frames} frames need correction")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user