$gridFieldConfig->addComponent($gridFieldForm = new GridFieldPopupForms());      
$gridFieldForm->setTemplate('CMSGridFieldPopupForms');

<?php

class SliderImage extends DataObject {

   static $db = array (
      'Title' => 'Varchar',
      'ImageLink' => 'Varchar'
   );

   static $has_one = array (
      'Home' => 'Home',
       'SliderMainImage' => 'Image'
   );

   public static $summary_fields = array(
   'Title' => 'Image Title',
   'ImageLink' => 'Link for the image'
   );

   public function getCMSFields_forPopup() {

    $imageField = new UploadField('SliderMainImage', 'Image');
    $imageField->allowedExtensions = array('jpg', 'png', 'gif');
   
    // Name, Description and Website fields
    return new FieldList(
       new TextField('Title', 'Title'),
       new TextareaField('ImageLink', 'Link for the image'),
       $thumbField
    );
   }
}

public static $has_many = array(
      'SliderImages' => 'SliderImage'
   );
   
   function getCMSFields() {
      $fields = parent::getCMSFields();
      
      
      $gridFieldConfig = GridFieldConfig::create()->addComponents(
    new GridFieldToolbarHeader(),
    new GridFieldAddNewButton('toolbar-header-right'),
    new GridFieldSortableHeader(),
    new GridFieldDataColumns(),
    new GridFieldPaginator(10),
    new GridFieldEditButton(),
    new GridFieldDeleteAction(),
    new GridFieldDetailForm()
    );
   
    $gridFieldConfig->addComponent($gridFieldForm = new GridFieldPopupForms());      
      $gridFieldForm->setTemplate('CMSGridFieldPopupForms');
   
    $gridField = new GridField("SliderImages", "Slider Images:", $this->SliderImages(), $gridFieldConfig);
    $fields->addFieldToTab("Root.Slider", $gridField);
      
      return $fields;
   }

Note: I'm currently working with an early pre release version of Silverstripe 3, and the code here is a start point and not everything works.