yjyoon / Raindrop_Detection star
윤영준 윤영준 2023-06-21
Hello YONA
@0ce9f58a794bfcd9aa73511bb949ece887df5bd5
 
0ce9f58
README.md (added)
+++ README.md
@@ -0,0 +1,1 @@
+# Pothhole-detection
 
0ce9f58
main.py (added)
+++ main.py
@@ -0,0 +1,57 @@
+import cv2
+import torch
+import numpy as np
+from ImageProcessing import MSCN
+from torchvision.io import read_image
+
+# Create a background subtractor object
+bg_subtractor = cv2.bgsegm.createBackgroundSubtractorGMG()
+kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(1,10))
+
+# Open video file or capture device
+video = cv2.VideoCapture('spring_-_38344 (Original).mp4')
+
+# Minimum and maximum area thresholds for filtering objects
+min_area = 1  # Adjust as needed
+max_area = 500  # Adjust as needed
+
+frame_count = 0
+while True:
+    # Read frame from video
+    ret, frame = video.read()
+    if not ret:
+        break
+
+    # Apply background subtraction
+    fg_mask = bg_subtractor.apply(frame)
+    fg_mask = cv2.morphologyEx(fg_mask, cv2.MORPH_OPEN, kernel)
+
+    # Apply thresholding to get binary foreground mask
+    thresh = cv2.threshold(fg_mask, 128, 255, cv2.THRESH_BINARY)[1]
+
+    # Find contours of the moving objects
+    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    # Filter objects by size
+    for contour in contours:
+        # Calculate contour area
+        area = cv2.contourArea(contour)
+
+        # Check if contour area is within the specified range
+        if min_area < area < max_area :
+            (x, y, w, h) = cv2.boundingRect(contour)
+            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
+
+    # Display the resulting frame
+
+    if frame_count > 120 :
+        cv2.imshow('Moving Object Detection', frame)
+        # cv2.imshow('movingobjectcontour', fg_mask)
+    # Exit if 'q' is pressed
+    frame_count += 1
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        break
+
+# Release the video capture and close windows
+video.release()
+cv2.destroyAllWindows()
 
0ce9f58
model/autoencoder.py (added)
+++ model/autoencoder.py
@@ -0,0 +1,134 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torchvision.models import vgg16
+
+class AutoEncoder(nn.Module):
+    def __init__(self):
+        super(AutoEncoder, self).__init__()
+
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=5, stride=1, padding=2, bias=False)
+        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False)
+
+        self.conv3 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv4 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=2, padding=1, bias=False)
+
+        self.conv5 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv6 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False)
+
+        self.dilated_conv1 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, dilation=2, padding=2, bias=False)
+        self.dilated_conv2 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, dilation=4, padding=4, bias=False)
+        self.dilated_conv3 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, dilation=8, padding=8, bias=False)
+        self.dilated_conv4 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, dilation=16, padding=16, bias=False)
+
+        self.conv7 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv8 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False)
+
+        self.deconv1 = nn.ConvTranspose2d(in_channels=256, out_channels=128, kernel_size=4, stride=2, padding=1, bias=False)
+        self.avg_pool1 = nn.AvgPool2d(kernel_size=3, stride=1, padding=1)
+
+        self.deconv2 = nn.ConvTranspose2d(in_channels=128, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
+        self.avg_pool2 = nn.AvgPool2d(kernel_size=3, stride=1, padding=1)
+
+        self.conv9 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv10 = nn.Conv2d(in_channels=64, out_channels=32, kernel_size=3, stride=1, padding=1, bias=False)
+
+        self.skip_output1 = nn.Conv2d(in_channels=256, out_channels=3, kernel_size=3, stride=1, padding=1, bias=False)
+        self.skip_output2 = nn.Conv2d(in_channels=128, out_channels=3, kernel_size=3, stride=1, padding=1, bias=False)
+        self.skip_output3 = nn.Conv2d(in_channels=32, out_channels=3, kernel_size=3, stride=1, padding=1, bias=False)
+
+    # maybe change it into concat Networks? this seems way to cumbersome.
+    def forward(self, input_tensor):
+        # Feed the input through each layer
+        relu1 = torch.relu(self.conv1(input_tensor))
+        relu2 = torch.relu(self.conv2(relu1))
+        relu3 = torch.relu(self.conv3(relu2))
+        relu4 = torch.relu(self.conv4(relu3))
+        relu5 = torch.relu(self.conv5(relu4))
+        relu6 = torch.relu(self.conv6(relu5))
+        relu7 = torch.relu(self.dilated_conv1(relu6))
+        relu8 = torch.relu(self.dilated_conv2(relu7))
+        relu9 = torch.relu(self.dilated_conv3(relu8))
+        relu10 = torch.relu(self.dilated_conv4(relu9))
+        relu11 = torch.relu(self.conv7(relu10))
+        relu12 = torch.relu(self.conv8(relu11))
+
+        deconv1 = self.deconv1(relu12)
+        avg_pool1 = self.avg_pool1(deconv1)
+        relu13 = torch.relu(avg_pool1)
+
+        relu14 = torch.relu(self.conv9(relu13 + relu3))
+
+        deconv2 = self.deconv2(relu14)
+        avg_pool2 = self.avg_pool2(deconv2)
+        relu15 = torch.relu(avg_pool2)
+
+        relu16 = torch.relu(self.conv10(relu15 + relu1))
+
+        skip_output_1 = self.skip_output1(relu12)
+        skip_output_2 = self.skip_output2(relu14)
+        skip_output_3 = torch.tanh(self.skip_output3(relu16))
+
+        ret = {
+            'skip_1': skip_output_1,
+            'skip_2': skip_output_2,
+            'skip_3': skip_output_3,
+        }
+
+        return ret
+
+
+class LossFunction(nn.Module):
+    def __init__(self):
+        super(LossFunction, self).__init__()
+
+        # Load pre-trained VGG model for feature extraction
+        self.vgg = vgg16(pretrained=True).features
+        self.vgg.eval()
+        for param in self.vgg.parameters():
+            param.requires_grad = False
+
+        self.lambda_i = [0.6, 0.8, 1.0]
+
+    def forward(self, input_tensor, label_tensor):
+        ori_height, ori_width = label_tensor.shape[2:]
+
+        # Rescale labels to match the scales of the outputs
+        label_tensor_resize_2 = F.interpolate(label_tensor, size=(ori_height // 2, ori_width // 2))
+        label_tensor_resize_4 = F.interpolate(label_tensor, size=(ori_height // 4, ori_width // 4))
+        label_list = [label_tensor_resize_4, label_tensor_resize_2, label_tensor]
+
+        # Initialize autoencoder model
+        autoencoder = AutoEncoder()
+        inference_ret = autoencoder(input_tensor)
+
+        output_list = [inference_ret['skip_1'], inference_ret['skip_2'], inference_ret['skip_3']]
+
+        # Compute lm_loss
+        lm_loss = 0.0
+        for index, output in enumerate(output_list):
+            mse_loss = nn.MSELoss()(output, label_list[index]) * self.lambda_i[index]
+            lm_loss += mse_loss
+
+        # Compute lp_loss
+        src_vgg_feats = self.extract_vgg_feats(label_tensor)
+        pred_vgg_feats = self.extract_vgg_feats(output_list[-1])
+
+        lp_losses = []
+        for index in range(len(src_vgg_feats)):
+            lp_losses.append(nn.MSELoss()(src_vgg_feats[index], pred_vgg_feats[index]))
+        lp_loss = torch.mean(torch.stack(lp_losses))
+
+        loss = lm_loss + lp_loss
+
+        return loss, inference_ret['skip_3']
+
+    def extract_vgg_feats(self, input_tensor):
+        # Extract features from the input tensor using the VGG network
+        feats = []
+        x = input_tensor
+        for layer_num, layer in enumerate(self.vgg):
+            x = layer(x)
+            if layer_num in {3, 8, 15, 22, 29}:
+                feats.append(x)
+        return feats(No newline at end of file)
 
0ce9f58
model/discriminator.py (added)
+++ model/discriminator.py
@@ -0,0 +1,1 @@
+from torch import nn(No newline at end of file)
 
0ce9f58
model/generator.py (added)
+++ model/generator.py
@@ -0,0 +1,250 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def conv3x3(in_ch, out_ch, stride=1, padding=1, groups=1, dilation=1):
+    return nn.Conv2d(in_ch, out_ch, kernel_size=3, stride=stride, padding=padding, groups=groups, dilation=dilation)
+
+
+def conv1x1(in_ch, out_ch, stride=1):
+    return nn.Conv2d(in_ch, out_ch, kernel_size=1, stride=stride)
+
+
+class ResNetBlock(nn.Module):
+    def __init__(self, blocks=3, layers=1, input_ch=3, out_ch=32, kernel_size=None, stride=1, padding=1, groups=1,
+                 dilation=1):
+        """
+        :type kernel_size: iterator or int
+        """
+        super(ResNetBlock, self).__init__()
+        if kernel_size is None:
+            kernel_size = [3, 3]
+        self.conv1 = nn.Conv2d(
+            input_ch, out_ch,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            groups=groups,
+            dilation=dilation
+        )
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(
+                out_ch, out_ch,
+                kernel_size=kernel_size,
+                stride=stride,
+                padding=padding,
+                groups=groups,
+                dilation=dilation
+            ),
+            nn.ReLU()
+        )
+        self.conv_hidden = nn.ModuleList()
+        for block in range(blocks):
+            for layer in range(layers):
+                self.conv_hidden.append(
+                    self.conv2
+                )
+        self.relu = nn.ReLU
+        self.blocks = blocks
+        self.layers = layers
+
+    def forward(self, x):
+        shortcut = x
+        x = self.conv1(shortcut)
+        for i, hidden_layer in enumerate(self.conv_hidden):
+            x = hidden_layer(x)
+            if (i % self.layers == 0) & (i != 0):
+                x = self.relu(x)
+                x += shortcut
+        return x
+
+
+class ConvLSTM(nn.Module):
+    def __init__(self, ch, kernel_size=3):
+        super(ConvLSTM, self).__init__()
+        self.padding = (kernel_size-1)/2
+        self.conv_i = nn.Conv2d(in_channels=ch, out_channels=ch, kernel_size=kernel_size, stride=1, padding=1,
+                                bias=False)
+        self.conv_f = nn.Conv2d(in_channels=ch, out_channels=ch, kernel_size=kernel_size, stride=1, padding=1,
+                                bias=False)
+        self.conv_c = nn.Conv2d(in_channels=ch, out_channels=ch, kernel_size=kernel_size, stride=1, padding=1,
+                                bias=False)
+        self.conv_o = nn.Conv2d(in_channels=ch, out_channels=ch, kernel_size=kernel_size, stride=1, padding=1,
+                                bias=False)
+        self.conv_attention_map = nn.Conv2d(in_channels=ch, out_channels=1, kernel_size=kernel_size, stride=1,
+                                            padding=1, bias=False)
+        self.ch = ch
+
+    def init_hidden(self, batch_size, image_size, init=0.5):
+        height, width = image_size
+        return torch.ones(batch_size, self.ch, height, width).to(self.conv_i.weight.device) * init
+
+    def forward(self, input_tensor, input_cell_state=None):
+        if input_cell_state is None:
+            batch_size, _, height, width = input_tensor.size()
+            input_cell_state = self.init_hidden(batch_size, (height, width))
+
+        conv_i = self.conv_i(input_tensor)
+        sigmoid_i = torch.sigmoid(conv_i)
+
+        conv_f = self.conv_f(input_tensor)
+        sigmoid_f = torch.sigmoid(conv_f)
+
+        cell_state = sigmoid_f * input_cell_state + sigmoid_i * torch.tanh(self.conv_c(input_tensor))
+
+        conv_o = self.conv_o(input_tensor)
+        sigmoid_o = torch.sigmoid(conv_o)
+
+        lstm_feats = sigmoid_o * torch.tanh(cell_state)
+
+        attention_map = self.conv_attention_map(lstm_feats)
+        attention_map = torch.sigmoid(attention_map)
+
+        return attention_map, cell_state, lstm_feats
+
+
+class GeneratorBlock(nn.Module):
+    def __init__(self, blocks=3, layers=1, input_ch=3, out_ch=32, kernel_size=None, stride=1, padding=1, groups=1,
+                 dilation=1):
+        """
+        :type kernel_size: iterator or int
+        """
+        super(GeneratorBlock, self).__init__()
+        if kernel_size is None:
+            kernel_size = [3, 3]
+        self.blocks = blocks
+        self.layers = layers
+        self.input_ch = input_ch
+        self.out_ch = out_ch
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.padding = padding
+        self.groups = groups
+        self.dilation = dilation
+        self.sigmoid = nn.Sigmoid()
+        self.resnet = nn.Sequential(
+            ResNetBlock(
+                blocks=self.blocks,
+                layers=self.layers,
+                input_ch=self.input_ch,
+                out_ch=self.out_ch,
+                kernel_size=self.kernel_size,
+                stride=self.stride,
+                padding=self.padding,
+                groups=self.groups,
+                dilation=self.dilation
+            )
+        )
+        self.LSTM = nn.Sequential(
+            ConvLSTM(
+                ch=out_ch, kernel_size=kernel_size,
+            )
+        )
+
+    def forward(self, original_image, prev_cell_state=None):
+        x = self.resnet(original_image)
+        attention_map, cell_state, lstm_feats = self.LSTM(x, prev_cell_state)
+        x = attention_map * original_image
+        return x, attention_map, cell_state, lstm_feats
+
+
+class Generator(nn.Module):
+    def __init__(self, repetition, blocks=3, layers=1, input_ch=3, out_ch=32, kernel_size=None, stride=1, padding=1,
+                 groups=1, dilation=1):
+        """
+        :type kernel_size: iterator or int
+        """
+        super(Generator, self).__init__()
+        if kernel_size is None:
+            kernel_size = [3, 3]
+        self.blocks = blocks
+        self.layers = layers
+        self.input_ch = input_ch
+        self.out_ch = out_ch
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.padding = padding
+        self.groups = groups
+        self.dilation = dilation
+        self.repetition = repetition
+        self.generator_block = nn.Sequential(
+            GeneratorBlock(blocks=blocks,
+                           layers=layers,
+                           input_ch=input_ch,
+                           out_ch=out_ch,
+                           kernel_size=kernel_size,
+                           stride=stride,
+                           padding=padding,
+                           groups=groups,
+                           dilation=dilation)
+        )
+        self.generator_blocks = nn.ModuleList()
+        for repetition in range(repetition):
+            self.conv_hidden.append(
+                self.generator_block
+            )
+
+    def forward(self, x):
+        cell_state = None
+        attention_map = None
+        for generator_block in self.generator_blocks:
+            x, attention_map, cell_state, lstm_feats = generator_block(x, cell_state)
+        return x, attention_map
+
+# need fixing
+class AttentiveRNNLoss(nn.Module):
+    def __init__(self):
+        super(AttentiveRNNLoss, self).__init__()
+
+    def forward(self, input_tensor, label_tensor):
+        # Initialize attentive rnn model
+        attentive_rnn = Generator
+        inference_ret = attentive_rnn(input_tensor)
+
+        loss = 0.0
+        n = len(inference_ret['attention_map_list'])
+        for index, attention_map in enumerate(inference_ret['attention_map_list']):
+            mse_loss = (0.8 ** (n - index + 1)) * nn.MSELoss()(attention_map, label_tensor)
+            loss += mse_loss
+
+        return loss, inference_ret['final_attention_map']
+
+# Need work
+class DiscriminativeNet(nn.Module):
+    def __init__(self, W, H):
+        super(DiscriminativeNet, self).__init__()
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=8, kernel_size=5, stride=1, padding=2)
+        self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=5, stride=1, padding=2)
+        self.conv3 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=2)
+        self.conv4 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2)
+        self.conv5 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=5, stride=1, padding=2)
+        self.conv6 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=5, stride=1, padding=2)
+        self.conv_map = nn.Conv2d(in_channels=128, out_channels=1, kernel_size=5, stride=1, padding=2, bias=False)
+        self.conv7 = nn.Conv2d(in_channels=128, out_channels=64, kernel_size=5, stride=4, padding=2)
+        self.conv8 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=4, padding=2)
+        self.conv9 = nn.Conv2d(in_channels=64, out_channels=32, kernel_size=5, stride=4, padding=2)
+        self.fc1 = nn.Linear(32 * W * H,
+                             1024)  # You need to adjust the input dimension here depending on your input size
+        self.fc2 = nn.Linear(1024, 1)
+
+    def forward(self, x):
+        x1 = F.leaky_relu(self.conv1(x))
+        x2 = F.leaky_relu(self.conv2(x1))
+        x3 = F.leaky_relu(self.conv3(x2))
+        x4 = F.leaky_relu(self.conv4(x3))
+        x5 = F.leaky_relu(self.conv5(x4))
+        x6 = F.leaky_relu(self.conv6(x5))
+        attention_map = self.conv_map(x6)
+        x7 = F.leaky_relu(self.conv7(attention_map * x6))
+        x8 = F.leaky_relu(self.conv8(x7))
+        x9 = F.leaky_relu(self.conv9(x8))
+        x9 = x9.view(x9.size(0), -1)  # flatten the tensor
+        fc1 = self.fc1(x9)
+        fc2 = self.fc2(fc1)
+        fc_out = torch.sigmoid(fc2)
+
+        # Ensure fc_out is not exactly 0 or 1 for stability of log operation in loss
+        fc_out = torch.clamp(fc_out, min=1e-7, max=1 - 1e-7)
+
+        return fc_out, attention_map, fc2
Add a comment
List