Generate Images from demo RealisticRendering

This ipython notebook can be found in client/examples/generate-images.ipynb. Before you start, please read the Tutorial: Getting Started first.

First, we need to load some python libraries for this tutorial.

import os, sys, time, re, json
import numpy as np
# from scipy import misc # for imread
# imread = misc.imread # use this instead of the imread in matplotlib
import cv2
imread = lambda x: cv2.imread(x)[:,:,(2,1,0)]

# Load matplotlib for plotting
import matplotlib.pyplot as plt
%matplotlib inline

Connect to the game

# Load unrealcv python client, do `pip install unrealcv` first.
from unrealcv import client
if not client.isconnected():
    print 'UnrealCV server is not running. Run the game downloaded from first.'
INFO:__init__:204:Got connection confirm: 'connected to RealisticRendering'
# You can use this to show help message of a function
# client?
# client.request?
# Test connection
res = client.request('vget /unrealcv/status')
print res
Is Listening
Client Connected

Read a camera trajectory from a file

Define a function to read a camera trajectory from a file

def read_camera_info(filename):
    with open(filename) as f:
        lines = f.readlines()
    # Parse camera location and rotation from file
    camera_pos = []
    for line_id in range(len(lines)):
        line = lines[line_id].strip() # Remove \n at the end
        if line_id % 3 == 0: # filename
        elif line_id % 3 == 1: # location
            location = [float(v) for v in line.split(' ')]
        elif line_id % 3 == 2: # Rotation
            rotation = [float(v) for v in line.split(' ')]
            camera_pos.append((location, rotation))
    return camera_pos

We will read a camera trajectory from realistic_rendering_camera_info.txt. This file contains a camera trajectory with 10 frames. The camera trajectory recorded using another python script.

!head -n 6 ./realistic_rendering_camera_info.txt
0.000 0.000 136.335
341.382 297.717 0.000
69.022 -11.622 126.405
345.077 72.858 0.000
camera_pos = read_camera_info('./realistic_rendering_camera_info.txt')
print camera_pos[0] # camera location and rotation
([0.0, 0.0, 136.335], [341.382, 297.717, 0.0])

Render images using the camera trajectory

Define a function to render an image and its annotation

def render_frame(client, cam_pose=None):
    ''' If you want to render a frame using current camera position, leave the pos argument to None '''
    if cam_pose is not None:
        # Set camera position
        loc = cam_pose[0] # location
        rot = cam_pose[1] # rotation
        cmd = 'vset /camera/0/location %.3f %.3f %.3f' % (loc[0], loc[1], loc[2])
        response = client.request(cmd)
        assert response == 'ok' # Make sure operation is successful
        cmd = 'vset /camera/0/rotation %.3f %.3f %.3f' % (rot[0], rot[1], rot[2])
        response = client.request(cmd)
        assert response == 'ok'

    f = dict(
        lit = client.request('vget /camera/0/lit'),
        depth = client.request('vget /camera/0/depth depth.exr'), # The depth is saved as HDR image
        object_mask = client.request('vget /camera/0/object_mask'),
        normal = client.request('vget /camera/0/normal'),
    return f

Define a utility function for plotting rendered images

def subplot_image(sub_index, image, param=None):
    if isinstance(image, str):
        image = imread(image)
    plt.imshow(image, param)

pos = camera_pos[0]
frame = render_frame(client, pos)

plt.rcParams['figure.figsize'] = (10.0, 7.0) # (w, h)
subplot_image(221, frame['lit'])
subplot_image(223, frame['object_mask'])
subplot_image(224, frame['normal'])

# depth is saved as HDR image and requires special care.
depth = cv2.imread(frame['depth'], cv2.IMREAD_ANYDEPTH)
subplot_image(222, depth, 'gray')

# for demostration purpose, we just render one frame. If you want to generate all frames of the trajectory
# simply do frames = [render_frame(client, pos) for pos in camera_pos]


Parse the object instance mask

Notice: this is a slow and very inefficient implementation. we are working on improving it. If you have suggestion about how to improve, please contact Weichao Qiu (

plt.rcParams['figure.figsize'] = (4, 3) # (w, h)
subplot_image(111, frame['object_mask'])

print '''Here we have the a color map of object instance, then we need to know the color for each object''' 
Here we have the a color map of object instance, then we need to know the color for each object


# Get a list of all objects in the scene
scene_objects = client.request('vget /objects').split(' ')
print 'There are %d objects in this scene' % len(scene_objects)
There are 295 objects in this scene
class Color(object):
    ''' A utility class to parse color value '''
    regexp = re.compile('\(R=(.*),G=(.*),B=(.*),A=(.*)\)')
    def __init__(self, color_str):
        self.color_str = color_str
        match = self.regexp.match(color_str)
        (self.R, self.G, self.B, self.A) = [int( for i in range(1,5)]

    def __repr__(self):
        return self.color_str

def subplot_color(index, color, title):
    im = np.zeros((100, 100, 4))
    # red = np.array([1.0, 0, 0, 1.0])
    color_array = np.array([color.R, color.G, color.B, color.A]) / 255.0 # repmat
    im = im + color_array
    print title, color

If we want to know the color of a specific object, we can use command vget /object/[str]/color. [str] is the object name. We first iterate all objects of the scene and get their color.

def get_color_mapping(client, object_list):
    ''' Get the color mapping for specified objects '''
    color_mapping = {}
    for objname in object_list:
        color_mapping[objname] = Color(client.request('vget /object/%s/color' % objname))
    return color_mapping

# Plot the annotation color for some objects. This cell might take about one minute
color_mapping = get_color_mapping(client, scene_objects)
selected_objects = ['WallPiece1_22', 'SM_Shelving_6', 'SM_Couch_1seat_5', 'SM_Frame_39']
plt.rcParams['figure.figsize'] = (12.0, 6.0) # (w, h)

baseindex = 100 + 10 * len(selected_objects) + 1
for i in range(len(selected_objects)):
    object_name = selected_objects[i]
    color = color_mapping[object_name]
    subplot_color(baseindex+i, color, object_name)
WallPiece1_22 (R=63,G=255,B=127,A=255)
SM_Shelving_6 (R=63,G=63,B=127,A=255)
SM_Couch_1seat_5 (R=255,G=0,B=255,A=255)
SM_Frame_39 (R=127,G=63,B=0,A=255)


Given we know the labeling color for each object, we can compute the object instance mask for each object.

def match_color(color_image, target_color, tolerance=3): # Tolerance is used to solve numerical issue
    match_region = np.ones(color_image.shape[0:2], dtype=bool)
    for c in range(3): # Iterate over three channels
        min_val = target_color[c]-tolerance; max_val = target_color[c]+tolerance
        channel_region = (color_image[:,:,c] >= min_val) & (color_image[:,:,c] <= max_val)
        # channel_region = color_image[:,:,c] == target_color[c]
        match_region &= channel_region
    return match_region

def compute_instance_mask(object_mask, color_mapping, objects):
    if isinstance(object_mask, str):
        object_mask = imread(object_mask)

    dic_instance_mask = {}
    for object_name in objects:
        color = color_mapping[object_name]
        region = match_color(object_mask, [color.R, color.G, color.B], tolerance=3)
        if region.sum() != 0: # Present in the image
            dic_instance_mask[object_name] = region
    return dic_instance_mask

# Compute the binary mask for each object instance
dic_instance_mask = compute_instance_mask(frame['object_mask'], color_mapping, scene_objects)
# dic_instance_mask[object_name] can return the binary mask of the object
# Do some correctness test
import ipynb_util
from ipynb_util import check_coverage
covered = check_coverage(dic_instance_mask)
# plt.imshow(covered)
Coverage 1.00
def plot_image_with_mask(image, mask):
    Mask is a binary matrix, only mask==1 will be plotted
    if isinstance(image, str):
        image = imread(image)
    subplot_image(121, image)
    masked_image = image.copy()
    masked_image[~mask] = 0
    subplot_image(122, masked_image)

# Load object category for each object
with open('object_category.json') as f:
    dic_objects_category = json.load(f)
categories = set(dic_objects_category.values())
print 'Num of objects in the scene:', len(scene_objects)
image_objects = dic_instance_mask.keys()
print 'Num of objects in this image:', len(image_objects)
print 'All objects in this image'
print '%20s : %s' % ('Category Name', 'Object Name')
for obj_type in categories:
    objects = [v for v in image_objects if dic_objects_category[v] == obj_type]
    if len(objects) > 6: # Trim the list if too long
        objects[6:] = []
    if len(objects) != 0:
        print '%20s : %s' % (obj_type, objects)
Num of objects in the scene: 295
Num of objects in this image: 18
All objects in this image
       Category Name : Object Name
                Wire : ['Wire1_17']
            Shelving : ['SM_Shelving_10']
               Couch : ['Couch_13']
                 Mug : ['Mug_30']
                Book : ['BookLP_182']
             Curtain : ['SM_Curtain_43', 'SM_Curtain_42', 'SM_Curtain_45']
              Statue : ['Statue_48']
          Trim_Floor : ['S_Trim_Floor_10']
              Carpet : ['Carpet_5']
           FloorLamp : ['SM_FloorLamp_5']
         EditorPlane : ['EditorPlane_33']
               Frame : ['SM_Frame_38']
           WallPiece : ['WallPiece6_32', 'WallPiece1_22']
        SlidingDoors : ['SlidingDoors_5']
         CoffeeTable : ['SM_CoffeeTable_14']

If we want to plot the region of an object instance

plot_image_with_mask(frame['lit'], dic_instance_mask['SM_DeskLamp_5'])

KeyError                                  Traceback (most recent call last)

<ipython-input-22-f644a35ee0f9> in <module>()
----> 1 plot_image_with_mask(frame['lit'], dic_instance_mask['SM_DeskLamp_5'])

KeyError: 'SM_DeskLamp_5'

If we want to plot the region of a category, such as Couch

def get_category_mask(category):
    mask = None
    for obj in image_objects:
        if dic_objects_category[obj] == category:
            if mask is None:
                mask = dic_instance_mask[obj]
                mask += dic_instance_mask[obj]
    if mask is None:
        print 'Error: the mask is empty'
    return mask

mask = get_category_mask('Couch')
plot_image_with_mask(frame['lit'], mask)
print 'The version of this ipython notebook:'
!git rev-parse --short HEAD
# store the py version of this file for version control
!jupyter nbconvert --to script generate-images.ipynb