Deep Dream

Generating Deep Dreams with Keras.

Introduction

“Deep dream” is an image-filtering technique which consists of taking an image classification model, and running gradient ascent over an input image to try to maximize the activations of specific layers (and sometimes, specific units in specific layers) for this input. It produces hallucination-like visuals.

It was first introduced by Alexander Mordvintsev from Google in July 2015.

Process:

• Load the original image.
• Define a number of processing scales (“octaves”), from smallest to largest.
• Resize the original image to the smallest scale.
• For every scale, starting with(the smallest (i$e. current one), { })
  • Run gradient ascent
  • Upscale image to the next scale
  • Reinject the detail that was lost at upscaling time
• Stop when we are back to the original size. To obtain the detail lost during upscaling, we simply take the original image, shrink it down, upscale it, and compare the result to the (resized) original image.

Setup

library(tensorflow)
library(keras)

base_image_path <- get_file("sky.jpg", "https://i.imgur.com/aGBdQyK.jpg")
result_prefix <- "sky_dream"

# These are the names of the layers
# for which we try to maximize activation,
# as well as their weight in the final loss
# we try to maximize.
# You can tweak these setting to obtain new visual effects.
layer_settings <- list(
  "mixed4" = 1.0,
  "mixed5" = 1.5,
  "mixed6" = 2.0,
  "mixed7" = 2.5
)

# Playing with these hyperparameters will also allow you to achieve new effects

step <- 0.01  # Gradient ascent step size
num_octave <- 3  # Number of scales at which to run gradient ascent
octave_scale <- 1.4  # Size ratio between scales
iterations <- 20  # Number of ascent steps per scale
max_loss <- 15.0

This is our base image:

plot_image <- function(img) {
  img %>%   
    as.raster(max = 255) %>% 
    plot()
}

base_image_path %>% 
  image_load() %>% 
  image_to_array() %>% 
  plot_image()

Let’s set up some image preprocessing/deprocessing utilities:

preprocess_image <- function(image_path) {
  # Util function to open, resize and format pictures
  # into appropriate arrays.
  img <- image_path %>% 
    image_load() %>% 
    image_to_array()
  dim(img) <- c(1, dim(img))
  inception_v3_preprocess_input(img)
}

deprocess_image <- function(x) {
  dim(x) <- dim(x)[-1]
  # Undo inception v3 preprocessing
  x <- x/2.0
  x <- x + 0.5
  x <- x*255.0
  x[] <- raster::clamp(as.numeric(x), 0, 255)
  x
}

Compute the Deep Dream loss

First, build a feature extraction model to retrieve the activations of our target layers given an input image.

# Build an InceptionV3 model loaded with pre-trained ImageNet weights
model <- application_inception_v3(weights = "imagenet", include_top = FALSE)

# Get the symbolic outputs of each "key" layer (we gave them unique names).
outputs_dict <- purrr::imap(layer_settings, function(v, name) {
  layer <- get_layer(model, name)
  layer$output
})

# Set up a model that returns the activation values for every target layer
# (as a dict)

feature_extractor <- keras_model(inputs = model$inputs, outputs = outputs_dict)

The actual loss computation is very simple:

compute_loss <- function(input_image) {
  features <- feature_extractor(input_image)
  # Initialize the loss
  loss <- tf$zeros(shape = shape())
  
  layer_settings %>% 
    purrr::imap(function(coeff, name) {
      activation <- features[[name]]
      scaling <- tf$reduce_prod(tf$cast(tf$shape(activation), "float32"))
      # We avoid border artifacts by only involving non-border pixels in the loss.
      coeff * tf$reduce_sum(tf$square(activation[, 3:-2, 3:-2, ])) / scaling
    }) %>% 
    purrr::reduce(tf$add)
}

Set up the gradient ascent loop for one octave

gradient_ascent_step <- tf_function(function(img, learning_rate) {
  with(tf$GradientTape() %as% tape, {    
    tape$watch(img)
    loss <- compute_loss(img)
  })
  
  # Compute gradients.
  grads <- tape$gradient(loss, img)
  # Normalize gradients.
  grads <- grads/tf$maximum(tf$reduce_mean(tf$abs(grads)), 1e-6)
  img <- img + learning_rate * grads
  list(loss, img)
})


gradient_ascent_loop <- function(img, iterations, learning_rate, max_loss = NULL) {
  for (i in seq_len(iterations)) {
    c(loss, img) %<-% gradient_ascent_step(img, learning_rate)
    if (!is.null(max_loss) && as.logical(loss > max_loss)) {
      break
    }
    cat("... Loss value at step ", i, ": ", as.numeric(loss), "\n")
  }
  img
}

Run the training loop, iterating over different octaves

original_img <- preprocess_image(base_image_path)
original_shape <- dim(original_img)[2:3]

successive_shapes <- list(original_shape)
for (i in seq_len(num_octave - 1)) {
  shape <- as.integer(original_shape / octave_scale^i)
  successive_shapes[[i+1]] <- shape
}
successive_shapes <- rev(successive_shapes)

shrunk_original_img <- tf$image$resize(original_img, successive_shapes[[1]])
img <- tf$identity(original_img)  # Make a copy
for (i in seq_along(successive_shapes)) {
  shape <- successive_shapes[[i]]
  
  cat("Processing octave ", i, "with shape:", shape, "\n")
  
  img <- tf$image$resize(img, shape)
  img <- gradient_ascent_loop(
    img, iterations = iterations, learning_rate = step, max_loss = max_loss
  )
  upscaled_shrunk_original_img <- tf$image$resize(shrunk_original_img, shape)
  same_size_original <- tf$image$resize(original_img, shape)
  lost_detail <- same_size_original - upscaled_shrunk_original_img
  
  img <- img + lost_detail
  shrunk_original_img <- tf$image$resize(original_img, shape)
}

Display the result.

img %>% 
  as.array() %>% 
  deprocess_image() %>% 
  plot_image()