import torch
from torch import nn
from torch.nn import functional as F
"""
Adopted from
"""
class FastGuidedFilterRefiner(nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
self.guilded_filter = FastGuidedFilter(1)
def forward_single_frame(self, fine_src, base_src, base_fgr, base_pha):
fine_src_gray = fine_src.mean(1, keepdim=True)
base_src_gray = base_src.mean(1, keepdim=True)
fgr, pha = self.guilded_filter(
torch.cat([base_src, base_src_gray], dim=1),
torch.cat([base_fgr, base_pha], dim=1),
torch.cat([fine_src, fine_src_gray], dim=1)).split([3, 1], dim=1)
return fgr, pha
def forward_time_series(self, fine_src, base_src, base_fgr, base_pha):
B, T = fine_src.shape[:2]
fgr, pha = self.forward_single_frame(
fine_src.flatten(0, 1),
base_src.flatten(0, 1),
base_fgr.flatten(0, 1),
base_pha.flatten(0, 1))
fgr = fgr.unflatten(0, (B, T))
pha = pha.unflatten(0, (B, T))
return fgr, pha
def forward(self, fine_src, base_src, base_fgr, base_pha, base_hid):
if fine_src.ndim == 5:
return self.forward_time_series(fine_src, base_src, base_fgr, base_pha)
else:
return self.forward_single_frame(fine_src, base_src, base_fgr, base_pha)
class FastGuidedFilter(nn.Module):
def __init__(self, r: int, eps: float = 1e-5):
super().__init__()
self.r = r
self.eps = eps
self.boxfilter = BoxFilter(r)
def forward(self, lr_x, lr_y, hr_x):
mean_x = self.boxfilter(lr_x)
mean_y = self.boxfilter(lr_y)
cov_xy = self.boxfilter(lr_x * lr_y) - mean_x * mean_y
var_x = self.boxfilter(lr_x * lr_x) - mean_x * mean_x
A = cov_xy / (var_x + self.eps)
b = mean_y - A * mean_x
A = F.interpolate(A, hr_x.shape[2:], mode='bilinear', align_corners=False)
b = F.interpolate(b, hr_x.shape[2:], mode='bilinear', align_corners=False)
return A * hr_x + b
class BoxFilter(nn.Module):
def __init__(self, r):
super(BoxFilter, self).__init__()
self.r = r
def forward(self, x):
# Note: The original implementation at
# uses faster box blur. However, it may not be friendly for ONNX export.
# We are switching to use simple convolution for box blur.
kernel_size = 2 * self.r + 1
kernel_x = torch.full((x.data.shape[1], 1, 1, kernel_size), 1 / kernel_size, device=x.device, dtype=x.dtype)
kernel_y = torch.full((x.data.shape[1], 1, kernel_size, 1), 1 / kernel_size, device=x.device, dtype=x.dtype)
x = F.conv2d(x, kernel_x, padding=(0, self.r), groups=x.data.shape[1])
x = F.conv2d(x, kernel_y, padding=(self.r, 0), groups=x.data.shape[1])
return x