Wrong colors when trying to refine a video with SDXL?

🧨 Diffusers
1

I’m trying to process a video file with the SDXL Refiner, however, the colors/brightness comes blown out. I don’t know if it is Diffusers related or if it is due to my limited understanding of the color processing code, but I would appreciate some help on this issue?
(Images from the videos and copy/paste assembled code to display the main parts of my code)

Output:

image
image1009×578 35.9 KB

import cv2
import numpy as np
from diffusers import StableDiffusionXLImg2ImgPipeline
from diffusers.utils import export_to_video

# Convert color space to RGB
def convert_to_srgb(frame):
    import cv2
    if frame.shape[2] == 1:
        frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
    elif frame.shape[2] == 3 and frame.shape[2] != cv2.COLOR_RGB2BGR:
        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    return frame


# Import the load_and_scale_video function
def load_and_scale_video(video_path, target_width=1024):
    
    cap = cv2.VideoCapture(video_path)

    if not cap.isOpened():
        raise IOError("Error opening video file")

    frames = []
    while True:
        ret, frame = cap.read()
        if not ret:
            break
        frame = convert_to_srgb(frame)
        
        # Calculate new height to maintain aspect ratio
        original_height, original_width, _ = frame.shape
        target_height = int((target_width / original_width) * original_height)

        # Resize the frame to be divisible by 8
        target_height -= target_height % 8
        target_width -= target_width % 8
        frame = cv2.resize(frame, (target_width, target_height))
        
        frames.append(frame)

    cap.release()
    return np.array(frames)

refine = StableDiffusionXLImg2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16)
refine.to("cuda")

# Load and scale the video clip
frames = load_and_scale_video(video_path)

video_frames = []
# Iterate through the frames
for frame_idx, frame in enumerate(frames):
    image = refine(prompt, image=frame).images[0]
    video_frames.append(image)
video_frames = np.array(video_frames)

export_to_video(video_frames)
2

If I save the images to png and then load them one by one with PIL it works.