[公开]网络安全专业化服务（04包：信息系统市级政务云应用监管）中标公告

021/Multi-modal-Self-supervised-Learning

/data/README.md
# data

• data/face: face image dataset
• data/clip: video clip dataset
• data/depth: depth image dataset
• data/motion: motion dataset

/data/depth/README.md
# data/depth

• data/depth/depth_images: depth image dataset
• data/depth/depth_gt: depth ground truth dataset

/data/face/README.md
# data/face

• data/face/face_images: face image dataset
• data/face/face_lmk: face landmark dataset
• data/face/face_mask: face mask dataset

/data/clip/README.md
# data/clip

• data/clip/clip_images: clip image dataset
• data/clip/clip_labels: clip label dataset

/data/motion/README.md
# data/motion

• data/motion/motion_images: motion image dataset
• data/motion/motion_labels: motion label dataset

/face/face_dataset.py
import os
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

def get_data_dir(data_dir):
return os.path.join(data_dir, "face_images")

class FaceDataset(object):

def __init__(self, data_dir):
self.data_dir = data_dir
self.data_path = get_data_dir(data_dir)

def get_data(self):
# Get all the image files in the data directory
all_image_files = os.listdir(self.data_path)
image_files = [os.path.join(self.data_path, file) for file in all_image_files if file.endswith('.jpg') or file.endswith('.png')]

# Load and preprocess the images
images = [self.preprocess_image(file) for file in image_files]

return images

def preprocess_image(self, image_file):
# Open the image using PIL
image = Image.open(image_file)
# Resize the image to 224x224 (or any other size you prefer)
image = image.resize((224, 224))
# Convert the image to RGB (if it's not already)
image = image.convert('RGB')
# Normalize the image pixel values to the range [0, 1]
image = image / 255.0
# Convert the image to a NumPy array
image = np.array(image)
# Add a batch dimension
image = image[np.newaxis, ...]
return image

/face/face_data.py
import os
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

def get_data_dir(data_dir):
return os.path.join(data_dir, "clip_images")

class ClipDataset(object):

def __init__(self, data_dir):
self.data_dir = data_dir
self.data_path = get_data_dir(data_dir)

def get_data(self):
# Get all the image files in the data directory
all_image_files = os.listdir(self.data_path)
image_files = [os.path.join(self.data_path, file) for file in all_image_files if file.endswith('.jpg') or file.endswith('.png')]

# Load and preprocess the images
images = [self.preprocess_image(file) for file in image_files]

return images

def preprocess_image(self, image_file):
# Open the image using PIL
image = Image.open(image_file)
# Resize the image to 224x224 (or any other size you prefer)
image = image.resize((224, 224))
# Convert the image to RGB (if it's not already)
image = image.convert('RGB')
# Normalize the image pixel values to the range [0, 1]
image = image / 255.0
# Convert the image to a NumPy array
image = np.array(image)
# Add a batch dimension
image = image[np.newaxis, ...]
return image

/clip/clip_dataset.py
import os
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from .face_dataset import FaceDataset
from .clip_dataset import ClipDataset

class MultiModalDataset(object):
def __init__(self, data_dir, mode='train'):
self.data_dir = data_dir
self.mode = mode

self.face_dataset = FaceDataset(os.path.join(self.data_dir, 'face_images'))
self.clip_dataset = ClipDataset(os.path.join(self.data_dir, 'clip_images'))

def get_data(self):
face_images = self.face_dataset.get_data()
clip_images = self.clip_dataset.get_data()

return face_images, clip_images

/data/motion/README.md
# data/motion

• data/motion/motion_images: motion image dataset
• data/motion/motion_labels: motion label dataset

/motion/motion_dataset.py
import os
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from .face_dataset import FaceDataset
from .motion_dataset import MotionDataset
from .clip_dataset import ClipDataset

class MultiModalDataset(object):
def __init__(self, data_dir, mode='train'):
self.data_dir = data_dir
self.mode = mode

self.face_dataset = FaceDataset(os.path.join(self.data_dir, 'face_images'))
self.clip_dataset = ClipDataset(os.path.join(self.data_dir, 'clip_images'))
self.motion_dataset = MotionDataset(os.path.join(self.data_dir, 'motion_images'))

def get_data(self):
face_images = self.face_dataset.get_data()
clip_images = self.clip_dataset.get_data()
motion