diff --git a/README-pandora.md b/README-pandora.md index 3d203af33..cba7ca975 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -3,6 +3,8 @@ ## 0. Setup ``` +docker run -it --rm -v /gel/usr/{USER_NAME}:/mnt/workspace/ -v /home-local2/{USER_NAME}.extra.nobkp/:/mnt/data/ --gpus '"device=0"' -p 8008:8008 -p 6006:6006 lantern_docker:latest /bin/bash + git clone https://github.com/MrKarimiD/nerfstudio-HDR.git conda activate nerfstudio-HDR export PYTHONPATH=$pwd:$PYTHONPATH @@ -137,9 +139,9 @@ ns-viewer --load-config outputs/scene_ns/lantern-nerfacto/2024-06-06_191338/conf - Go to RENDER sub-menu. - Clic on ADD CAMERA to manualy set camera view points. - Clic on RENDER to get the command. - ``` - ns-render camera-path --load-config outputs/scene_ns/lantern-nerfacto/2024-06-06_191338/config.yml --camera-path-filename /mnt/data/scene/scene_ns/camera_paths/2024-06-06_191338.json --output-path /mnt/data/scene/renders/2024-06-06_191338.mp4 - ``` + ``` + ns-render camera-path --load-config outputs/scene_ns/lantern-nerfacto/2024-06-06_191338/config.yml --camera-path-filename /mnt/data/scene/scene_ns/camera_paths/2024-06-06_191338.json --output-path /mnt/data/scene/renders/2024-06-06_191338.mp4 + ``` - To render a video fast-exposed: add --rendered-output-names rgb_fast ``` ns-render camera-path --load-config outputs/scene_ns/lantern-nerfacto/2024-06-06_191338/config.yml --camera-path-filename /mnt/data/scene/scene_ns/camera_paths/2024-06-06_191338.json --output-path /mnt/data/scene/renders/2024-06-06_191338.mp4 --rendered-output-names rgb_fast diff --git a/lantern_scripts/aligning_interface/align_auto.py b/lantern_scripts/aligning_interface/align_auto.py new file mode 100644 index 000000000..405d8e426 --- /dev/null +++ b/lantern_scripts/aligning_interface/align_auto.py @@ -0,0 +1,212 @@ +import cv2 +from matplotlib import pyplot as plt +import numpy as np +import os + +class ContourTooSmallException(Exception): + pass + +class Image: + def __init__(self, image, threshold = 5): + self.image = image.copy() + self.threshold = threshold + self.gray = cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY) + _, self.thresholded = cv2.threshold(self.gray, threshold, 255, cv2.THRESH_BINARY) + self.contours, _ = cv2.findContours(self.thresholded, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + +class Contour: + def __init__(self, contour, well_image_threshold_index = None, min_area = 50.0): + self.contour = contour + self.area = cv2.contourArea(contour) + if self.area < min_area: + raise ContourTooSmallException(f"Contour area {self.area} is too small.") + self.centroid = np.mean(contour, axis=0)[0] + self.relative_position = None + self.match = None + self.score = float('inf') + self.well_image_threshold_index = well_image_threshold_index + + def overlap(self, contour): + image_shape = (960, 960) + mask1 = np.zeros(image_shape, dtype=np.uint8) + mask2 = np.zeros(image_shape, dtype=np.uint8) + cv2.drawContours(mask1, [self.contour], -1, 255, thickness=cv2.FILLED) + cv2.drawContours(mask2, [contour.contour], -1, 255, thickness=cv2.FILLED) + overlap_mask = cv2.bitwise_and(mask1, mask2) + overlap_area = np.sum(overlap_mask) / 255 + return overlap_area > 0 + + def get_matching_score(self, contour): + dist_shape = cv2.matchShapes(self.contour, contour.contour, cv2.CONTOURS_MATCH_I2, 0.0) + dist_spatial = np.linalg.norm(np.array(self.centroid) - np.array(contour.centroid)) + return 0.7 * dist_shape + 0.3 * dist_spatial + + def match_contour(self, contour, score): + self.match = contour + self.score = score + + def unmatch_contour(self): + self.match = None + self.score = float('inf') + + def get_points(self): + return self.contour[:, 0, :] + + +class AlignImages: + def __init__(self, well_image, fast_image): + self.fast_image = Image(fast_image) + self.fast_contours = [] + for contour in self.fast_image.contours: + try: + contour_obj = Contour(contour) + self.fast_contours.append(contour_obj) + except ContourTooSmallException as e: + continue + self.matched_well_contours = [None] * len(self.fast_contours) + self.well_image = Image(well_image) + self.well_images = [] + self.well_thresholds = (230, 235, 240, 245, 250) + for threshold in self.well_thresholds: + self.well_images.append(Image(well_image, threshold)) + self.image_height, self.image_width = self.fast_image.image.shape[:2] + + def all_contours_matched(self): + for contour in self.fast_contours: + if contour.match == None: + return False + return True + + def existing_contour_match(self, contour): + for i, existing_contour in enumerate(self.matched_well_contours): + if existing_contour != None and contour.overlap(existing_contour): + return i, self.fast_contours[i] + return None, None + + def delete_unmatched_contours(self): + for contour_index in range(len(self.fast_contours) -1, -1, -1): + if self.fast_contours[contour_index].match == None or self.matched_well_contours[contour_index] == None: + del self.fast_contours[contour_index] + del self.matched_well_contours[contour_index] + + def match_contours(self): + iterations = 0 + while not self.all_contours_matched(): + if iterations >= 5: + self.delete_unmatched_contours() + break + for contour_index, fast_contour in enumerate(self.fast_contours): + for threshold_index, _ in enumerate(self.well_thresholds): + for well_contour in self.well_images[threshold_index].contours: + try: + well_contour = Contour(well_contour, threshold_index) + score = fast_contour.get_matching_score(well_contour) + if score < fast_contour.score: + if score >= 200: + continue + match_index, match = self.existing_contour_match(well_contour) + if match != None: + if score < match.score: + match.unmatch_contour() + self.matched_well_contours[match_index] = None + fast_contour.match_contour(well_contour, score) + self.matched_well_contours[contour_index] = well_contour + else: + continue + else: + fast_contour.match_contour(well_contour, score) + self.matched_well_contours[contour_index] = well_contour + except ContourTooSmallException as e: + continue + iterations += 1 + return list(zip(self.matched_well_contours, self.fast_contours)) + + def get_point_descriptors(self, points, image): + sift = cv2.SIFT_create() + keypoints = [cv2.KeyPoint(x=float(p[0]), y=float(p[1]), size=20) for p in points] + keypoints, descriptors = sift.compute(image, keypoints) + return descriptors + + def match_points(self, well_contour, fast_contour): + well_points = well_contour.get_points() + fast_points = fast_contour.get_points() + well_descriptors = self.get_point_descriptors(well_points, self.well_images[well_contour.well_image_threshold_index].thresholded) + fast_descriptors = self.get_point_descriptors(fast_points, self.fast_image.thresholded) + bf = cv2.BFMatcher() + matches = bf.match(fast_descriptors, well_descriptors) + matches = sorted(matches, key=lambda x: x.distance) + fast_matched_points = [fast_points[m.queryIdx] for m in matches] + well_matched_points = [well_points[m.trainIdx] for m in matches] + return well_matched_points, fast_matched_points + + def alignImages(self): + matched_contours = self.match_contours() + # show_contours(self.well_images[5].thresholded, self.well_images[4].thresholded, matched_contours) + well_points_combined = [] + fast_points_combined = [] + for well_contour, fast_contour in matched_contours: + well_matched_points, fast_matched_points = self.match_points(well_contour, fast_contour) + well_points_combined.extend(well_matched_points) + fast_points_combined.extend(fast_matched_points) + # show_points(well_image, fast_image, well_points_combined, fast_points_combined) + if len(well_points_combined) >= 4: + well_points = np.float32(well_points_combined) + fast_points = np.float32(fast_points_combined) + H, mask = cv2.findHomography(fast_points, well_points, cv2.RANSAC, 5.0) + warped_image = cv2.warpPerspective(self.fast_image.image, H, (self.image_width, self.image_height)) + else: + print("Not enough points for homography.") + return warped_image, H + +def show_contours(well_image, fast_image, matched_contours): + well_image_c = well_image.copy() + fast_image_c = fast_image.copy() + + for well_contour, fast_contour in matched_contours: + color = tuple(np.random.randint(0, 256, 3).tolist()) + cv2.drawContours(well_image_c, well_contour.contour, -1, color, 2) + cv2.drawContours(fast_image_c, fast_contour.contour, -1, color, 2) + + fig, axes = plt.subplots(1, 2, figsize=(120, 120), gridspec_kw={'wspace': 0, 'hspace': 0}) + fig.subplots_adjust(left=0, right=1, bottom=0, top=1) + axes[1].imshow(well_image_c) + axes[1].axis('off') + axes[0].imshow(fast_image_c) + axes[0].axis('off') + + fig.canvas.draw() + plt.show() + +def show_points(well_image, fast_image, well_points, fast_points): + well_image_c = well_image.copy() + fast_image_c = fast_image.copy() + + for p1 in well_points: + cv2.circle(well_image_c, tuple(p1), 2, (0, 255, 0), -1) + + for p2 in fast_points: + cv2.circle(fast_image_c, tuple(p2), 2, (0, 255, 0), -1) + + fig, axes = plt.subplots(1, 2, figsize=(20, 10)) + axes[0].imshow(well_image_c) + axes[0].axis('off') + axes[0].set_title('Well-exposed Image with Points') + axes[1].imshow(fast_image_c) + axes[1].axis('off') + axes[1].set_title('Aligned Fast-exposed Image') + plt.show() + +def align(well_image, fast_image): + align_images = AlignImages(well_image, fast_image) + return align_images.alignImages() + +if __name__ == '__main__': + index = 0 + path_well = '...' + path_fast = '...' + well_images = sorted(os.listdir(path_well)) + fast_images = sorted(os.listdir(path_fast)) + well_image = cv2.cvtColor(cv2.imread(os.path.join(path_well, well_images[index])), cv2.COLOR_BGR2RGB) + fast_image = cv2.cvtColor(cv2.imread(os.path.join(path_fast, fast_images[index])), cv2.COLOR_BGR2RGB) + + warped_image, H = align(well_image, fast_image) \ No newline at end of file diff --git a/lantern_scripts/aligning_interface/align_manual.py b/lantern_scripts/aligning_interface/align_manual.py new file mode 100644 index 000000000..05f9301a3 --- /dev/null +++ b/lantern_scripts/aligning_interface/align_manual.py @@ -0,0 +1,45 @@ +import cv2 +from matplotlib import pyplot as plt +import matplotlib.patches as patches +import numpy as np + +def capture_points_on_image(fig, ax, n): + plt.sca(ax) + pts = plt.ginput(n, timeout=0) + + for e in range(len(pts)): + ax.add_patch(patches.Circle((pts[e][0], pts[e][1]), 1.0, edgecolor='r', facecolor='none')) + ax.text(pts[e][0], pts[e][1], e, color = 'r') + + fig.canvas.draw() + return pts + +def align_manual(im, im_HDR, n): + im_mod = im.copy() + im_HDR_mod = im_HDR.copy() + + # Show images to select points + fig, axes = plt.subplots(1, 2, figsize=(120, 120), gridspec_kw={'wspace': 0, 'hspace': 0}) + fig.subplots_adjust(left=0, right=1, bottom=0, top=1) + axes[0].imshow(im_HDR_mod) + axes[0].axis('off') + axes[1].imshow(im_mod) + axes[1].axis('off') + + # Manually select points + im_HDR_pts = capture_points_on_image(fig, axes[0], n) + im_pts = capture_points_on_image(fig, axes[1], n) + + # plt.savefig('points.png') + plt.close() + + # Warp image + height, width = im.shape[:2] + H, sattus = cv2.findHomography(np.float32(im_HDR_pts), np.float32(im_pts)) + warped_image = cv2.warpPerspective(im_HDR, H, (width, height)) + + alpha = 50.0 / 100.0 + beta = 1.0 - alpha + merged_image = cv2.addWeighted(im, beta, warped_image, alpha, 0) + + return warped_image, H \ No newline at end of file diff --git a/lantern_scripts/aligning_interface/main.py b/lantern_scripts/aligning_interface/main.py new file mode 100644 index 000000000..415414f46 --- /dev/null +++ b/lantern_scripts/aligning_interface/main.py @@ -0,0 +1,149 @@ +import json +import os +import cv2 +from PIL import Image, ImageTk +import tkinter as tk +from tkinter import Scale + +import numpy as np +from align_auto import align +from align_manual import align_manual + +class ImageTransparencyAdjuster: + def __init__(self, master): + self.master = master + + self.path_well = '...' + self.path_fast = '...' + self.path_transformed = '...' + self.path_merged = '...' + self.path_masks = '...' + self.json_file = '...' + + os.makedirs(self.path_transformed, exist_ok=True) + os.makedirs(self.path_merged, exist_ok=True) + + self.well_images = sorted(os.listdir(self.path_well)) + self.fast_images = sorted(os.listdir(self.path_fast)) + self.index = 82 + self.matrix = np.zeros((3, 3)) + self.merged_image = None + self.mask = None + + self.master.title("Image Transparency Adjuster") + + self.well, self.fast, self.warped_image = self.load_image_set(self.index) + + frame = tk.Frame(master) + frame.pack(side=tk.LEFT) + + self.label = tk.Label(frame) + self.label.pack() + + slider = Scale(master, from_=0, to=100, orient=tk.HORIZONTAL, command=self.update_image) + slider.set(50) + slider.pack(side=tk.RIGHT, fill=tk.Y) + + self.master.bind('', self.handle_key) + + self.update_image(50) + + def load_image_set(self, index): + well = cv2.cvtColor(cv2.imread(os.path.join(self.path_well, self.well_images[index])), cv2.COLOR_BGR2RGB) + fast = cv2.cvtColor(cv2.imread(os.path.join(self.path_fast, self.fast_images[index])), cv2.COLOR_BGR2RGB) + + try: + warped_image = cv2.cvtColor(cv2.imread(os.path.join(self.path_transformed, self.fast_images[index])), cv2.COLOR_BGR2RGB) + except Exception: + print("Transformed image not found:", self.index, self.fast_images[self.index]) + warped_image = fast + return well, fast, warped_image + + def update_image(self, alpha): + alpha = float(alpha) / 100.0 + beta = 1.0 - alpha + self.merged_image = cv2.addWeighted(self.well, beta, self.warped_image, alpha, 0) + merged_image = Image.fromarray(self.merged_image) + merged_image_tk = ImageTk.PhotoImage(merged_image) + self.label.config(image=merged_image_tk) + self.label.image = merged_image_tk + self.first_run = False + self.master.update() + self.master.focus_force() + + def transform(self, manual, n): + if manual: + self.warped_image, self.matrix = align_manual(self.well, self.fast, n) + else: + try: + self.warped_image, self.matrix = align(self.well, self.fast) + print("Aligned!") + except Exception as e: + print("Failed to align.", e) + self.update_image(50) + + def save_transformation(self): + try: + with open(self.json_file, 'r') as f: + data = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + data = {'matrices': []} + + found = False + for item in data['matrices']: + if item['name'] == self.fast_images[self.index]: + item['matrix'] = self.matrix.tolist() + found = True + break + + if not found: + named_matrix = {'name': self.fast_images[self.index], 'matrix': self.matrix.tolist()} + data['matrices'].append(named_matrix) + + with open(self.json_file, 'w') as f: + json.dump(data, f, indent=4) + + def delete_transformation(self): + try: + with open(self.json_file, 'r') as f: + data = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + data = {'matrices': []} + + data['matrices'] = [item for item in data['matrices'] if item['name'] != self.fast_images[self.index]] + + with open(self.json_file, 'w') as f: + json.dump(data, f, indent=4) + + def handle_key(self, event): + if event.keysym in ['Right', 'Left']: + self.matrix = np.zeros((3, 3)) + self.index += 1 if event.keysym == 'Right' else -1 + self.well, self.fast, self.warped_image = self.load_image_set(self.index) + self.update_image(50) + print(self.index, self.fast_images[self.index]) + elif event.keysym in ['Up', 'Down']: + self.matrix = np.zeros((3, 3)) + self.warped_image = self.fast if event.keysym == 'Down' else cv2.cvtColor(cv2.imread(os.path.join(self.path_transformed, self.fast_images[self.index])), cv2.COLOR_BGR2RGB) + self.update_image(50) + elif event.keysym == 's': + cv2.imwrite(os.path.join(self.path_transformed, self.fast_images[self.index]), cv2.cvtColor(self.warped_image, cv2.COLOR_RGB2BGR)) + cv2.imwrite(os.path.join(self.path_merged, self.fast_images[self.index]), cv2.cvtColor(self.merged_image, cv2.COLOR_RGB2BGR)) + self.save_transformation() + print('Saved:', self.index, self.fast_images[self.index]) + elif event.keysym == 'd': + try: + os.remove(os.path.join(self.path_transformed, self.fast_images[self.index])) + self.delete_transformation() + print('Deleted:', self.index, self.fast_images[self.index]) + except (FileNotFoundError): + print('File not found:', self.index, self.fast_images[self.index]) + else: + pts = {'a': 0, '4': 4, '6': 6, '8': 8, '1': 10} + if event.char in pts: + self.transform(event.char != 'a', pts[event.char]) + +if __name__ == "__main__": + root = tk.Tk() + app = ImageTransparencyAdjuster(root) + root.mainloop()