    def __contains__(self, value):
        config = self.configuration(revert_to_file=True)
        # Accesses the subarray READONLY
        for partition in self.partitions.matrix.flat:
            partition.open(config)
            array = partition.array()
            partition.close()
            if value in array:
                return True
        #--- End: for

        return False
    #--- End: def

    def __getitem__(self, indices):
        '''

Implement indexing

x.__getitem__(indices) <==> x[indices]

'''
        size = self._size
        if size == 1 or indices is Ellipsis:
            return self.copy()

        d = self

        axes        = d._axes
        flip        = d._flip
        shape       = d._shape
        cyclic_axes = d._cyclic
            
        try:
            arg = indices[0]
        except (IndexError, TypeError):
            envelope = False
        else:     
            envelope = arg == 'envelope' 
            if envelope or arg == 'compress':
                auxiliary_mask = indices[1]
                indices = indices[2:]                
            else:
                auxiliary_mask = None
        #--- End: try

        parsed_indices = parse_indices(shape, indices, True, True, envelope)
        if envelope:
            indices, roll, flip_axes, compressed_indices = parsed_indices
        else:            
            indices, roll, flip_axes = parsed_indices

        if roll:
            for axis, shift in roll.iteritems():
                if axes[axis] not in cyclic_axes:
                    raise IndexError(
"Can't take a cyclic slice of a non-cyclic axis (axis {0})".format(axis))

                d = d.roll(axis, shift)
        #--- End: if

        new_shape = tuple(map(_size_of_index, indices, shape))
        new_size  = long(reduce(operator_mul, new_shape, 1))

        new = Data.__new__(Data)

        new._shape      = new_shape
        new._size       = new_size
        new._ndim       = d._ndim
        new._fill_value = d._fill_value
        new._hardmask   = d._hardmask
        new._isdt       = d._isdt
        new._all_axes   = d._all_axes
        new._flatten    = d._flatten
        new._cyclic     = cyclic_axes
        new._axes       = axes
        new._flip       = flip
        new._dtype      = d.dtype
        new.Units       = d.Units

        partitions = d.partitions
    
        new_partitions = PartitionMatrix(_overlapping_partitions(partitions, 
                                                                 indices, 
                                                                 axes,
                                                                 flip),
                                         partitions.axes)

        if new_size != size:
            new_partitions.set_location_map(axes)

        new.partitions = new_partitions

        # ------------------------------------------------------------
        # Index an existing auxiliary mask
        # ------------------------------------------------------------
        if not d._auxiliary_mask:
            new._auxiliary_mask = None
        else:
            new_auxiliary_mask = []
            for m in d._auxiliary_mask:
                # Note: By now indices is strictly monotonically
                #       increasing and doesn't roll.
                mask_indices = [(slice(None) if n == 1 else index)
                                for n, index in zip(m._shape, indices)]
                new_auxiliary_mask.append(m[tuple(mask_indices)])
            #--- End: for
            new._auxiliary_mask = new_auxiliary_mask
        #--- End: if

        # --------------------------------------------------------
        # Apply the input auxiliary mask
        # --------------------------------------------------------
        if auxiliary_mask:
            auxiliary_mask = [m.copy() for m in auxiliary_mask]
            if new._auxiliary_mask:
                new._auxiliary_mask.extend(auxiliary_mask)
            else:
                new._auxiliary_mask = auxiliary_mask
        #--- End: if

        if envelope:
            # --------------------------------------------------------
            # Mask out unselected points within the envelope
            # --------------------------------------------------------
            mask = self._auxiliary_mask_from_1d_indices(compressed_indices)
            if mask:
                if new._auxiliary_mask:
                    new._auxiliary_mask.extend(mask)
                else:
                    new._auxiliary_mask = mask
        #--- End: if

        # ------------------------------------------------------------
        # Flip axes
        # ------------------------------------------------------------
        if flip_axes:
            new.flip(flip_axes, i=True)

        # ------------------------------------------------------------
        # Mark cyclic axes which have been reduced in size as
        # non-cyclic
        # ------------------------------------------------------------
        if cyclic_axes:
            x = [i
                 for i, (axis, n0, n1) in enumerate(zip(axes, shape, new_shape))
                 if n1 != n0 and axis in cyclic_axes]
            if x:
                new.cyclic(x, iscyclic=False)
        #--- End: if

        # ------------------------------------------------------------
        # Remove size 1 axes from the partition matrix
        # ------------------------------------------------------------
        new_partitions.squeeze(i=True)

        return new
    #--- End: def

    def __setitem__(self, indices, value):
        '''

Implement indexed assignment

x.__setitem__(indices, y) <==> x[indices]=y

Assignment to data array elements defined by indices.

Elements of a data array may be changed by assigning values to a
subspace. See `__getitem__` for details on how to define subspace of
the data array.

**Missing data**

The treatment of missing data elements during assignment to a subspace
depends on the value of the `hardmask` attribute. If it is True then
masked elements will notbe unmasked, otherwise masked elements may be
set to any value.

In either case, unmasked elements may be set, (including missing
data).

Unmasked elements may be set to missing data by assignment to the
`cf.masked` constant or by assignment to a value which contains masked
elements.

.. seealso:: `cf.masked`, `hardmask`, `where`

:Examples:

'''            
        def _mirror_slice(index, size):
            '''

    Return a slice object which creates the reverse of the input
    slice.

    The step of the input slice must have a step of `.
    
    :Parameters:
    
        index : slice
            A slice object with a step of 1.

        size : int
    
    :Returns:
    
        out : slice
    
    :Examples:
    
    >>> s = slice(2, 6)
    >>> t = _mirror_slice(s, 8)
    >>> s, t
    slice(2, 6), slice(5, 1, -1)
    >>> range(8)[s]
    [2, 3, 4, 5]
    >>> range(8)[t]
    [5, 4, 3, 2]
    >>> range(7, -1, -1)[s]
    [5, 4, 3, 2]
    >>> range(7, -1, -1)[t]
    [2, 3, 4, 5]
    
    '''
            start, stop, step = index.indices(size)
            size -= 1
            start = size - start
            stop  = size - stop
            if stop < 0:
                stop = None
                
            return slice(start, stop, -1)
        #--- End: def

#        pda_args = self.pda_args()
        pda_args = self.configuration()

        # ------------------------------------------------------------
        # parse the indices
        # ------------------------------------------------------------
        indices_in = indices
#        indices, roll, flip_axes = _parse_indices(self._shape, indices_in)
        indices, roll, flip_axes = parse_indices(self._shape, indices_in, True, True)

        if roll:
            for iaxis, shift in roll.iteritems():
                self.roll(iaxis, shift, i=True)

        scalar_value = False
        if value is cf_masked:
            scalar_value  = True
        else:
            copied = False
            if not isinstance(value, Data):
                # Convert to the value to a Data object
                value = type(self)(value, self.Units)
            else:
                if value.Units.equivalent(self.Units):
                    if not value.Units.equals(self.Units):                    
                        value = value.copy()                    
                        value.Units = self.Units
                        copied = True
                elif not value.Units:                    
                    value = value.override_units(self.Units)
                    copied = True
                else:
                    raise ValueError("Some bad units %r, %r" %
                                     (self.Units, value.Units))
            #--- End: if

            value2dt = self._isdt and not value._isdt

            if value._size == 1:
                scalar_value = True                
                if value2dt:
                    if not copied:
                        value = value.copy()
                    value.asdatetime()
                #--- End: if
                value = value.datum(0)
        #--- End: if

        if scalar_value:
            # --------------------------------------------------------
            # The value is logically scalar
            # --------------------------------------------------------           
            for partition in self.partitions.matrix.flat:
                p_indices, shape = partition.overlaps(indices)
                if p_indices is None:
                    # This partition does not overlap the indices
                    continue

                partition.open(pda_args)
                array = partition.array() #**pda_args)

                if value is cf_masked and not partition.masked:
                    # The assignment is masking elements, so turn a
                    # non-masked sub-array into a masked one.
                    array = array.view(numpy_ma_MaskedArray)
                    partition.subarray = array

                set_subspace(array, p_indices, value)
                partition.close()
            #--- End: for

            if roll:
                for iaxis, shift in roll.iteritems():
                    self.roll(iaxis, -shift, i=True)
            #--- End: if

            return
        #--- End: if

        # ------------------------------------------------------------
        # Still here? Then the value is not logically scalar.
        # ------------------------------------------------------------
        data0_shape  = self._shape
        value_shape = value._shape
        
        shape0 = map(_size_of_index, indices, data0_shape)
        self_ndim  = self._ndim
        value_ndim = value._ndim
        align_offset = self_ndim - value_ndim
        if align_offset >= 0:
            # self has more dimensions than other
            shape0   = shape0[align_offset:]
            shape1   = value_shape
            ellipsis = None 
            
            flip_axes = [i-align_offset for i in flip_axes
                         if i >= align_offset]                
        else:
            # value has more dimensions than self
            v_align_offset = -align_offset
            if value_shape[:v_align_offset] != [1] * v_align_offset:
                # Can only allow value to have more dimensions then
                # self if the extra dimensions all have size 1.
                raise ValueError("Can't broadcast shape %r across shape %r" %
                                 (value_shape, data0_shape))
            
            shape1       = value_shape[v_align_offset:]
            ellipsis     = Ellipsis
            align_offset = 0
        #--- End: if

        # Find out which of the dimensions of value are to be
        # broadcast, and those which are not. Note that, as in numpy,
        # it is not allowed for a dimension in value to be larger than
        # a size 1 dimension in self
        base_value_indices       = []
        non_broadcast_dimensions = []

        for i, (a, b) in enumerate(izip(shape0, shape1)):
            if b == 1:
                base_value_indices.append(slice(None))
            elif a == b and b != 1:
                base_value_indices.append(None)
                non_broadcast_dimensions.append(i)
            else:
                raise ValueError("Can't broadcast shape %r across shape %r" %
                                 (shape1, shape0))
        #--- End: for

        previous_location = ((-1,),) * self_ndim
        start             = [0] * value_ndim

#        save = pda_args['save']
        keep_in_memory = pda_args['keep_in_memory']

        value.to_memory()

        for partition in self.partitions.matrix.flat:
            p_indices, shape = partition.overlaps(indices)        

            if p_indices is None:
                # This partition does not overlap the indices          
                continue

            # --------------------------------------------------------
            # Find which elements of value apply to this partition
            # --------------------------------------------------------
            value_indices = base_value_indices[:]
       
            for i in non_broadcast_dimensions:
                j                  = i + align_offset
                location           = partition.location[j][0]
                reference_location = previous_location[j][0]

                if location > reference_location:
                    stop             = start[i] + shape[j]
                    value_indices[i] = slice(start[i], stop)
                    start[i]         = stop

                elif location == reference_location:
                    value_indices[i] = previous_slice[i]

                elif location < reference_location:
                    stop             = shape[j]
                    value_indices[i] = slice(0, stop)
                    start[i]         = stop
            #--- End: for

            previous_location = partition.location
            previous_slice    = value_indices[:]
          
            for i in flip_axes:
                value_indices[i] = _mirror_slice(value_indices[i], shape1[i])

            if ellipsis:
                value_indices.insert(0, ellipsis)

            # --------------------------------------------------------
            # 
            # --------------------------------------------------------
            if value2dt:
                v = value[tuple(value_indices)].dtarray
            else:
                v = value[tuple(value_indices)].varray

            if keep_in_memory:
                v = v.copy()

            # Update the partition's data
            partition.open(pda_args)
            array = partition.array() #**pda_args)

            if not partition.masked and numpy_ma_isMA(v):
                # The sub-array is not masked and the assignment is
                # masking elements, so turn the non-masked sub-array
                # into a masked one.
                array = array.view(numpy_ma_MaskedArray)
                partition.subarray = array
            #--- End: if

            set_subspace(array, p_indices, v)

            partition.close()
        #--- End: for

        if roll:
            # Unroll
            for iaxis, shift in roll.iteritems():
                self.roll(iaxis, -shift, i=True)
        #--- End: if        
    #--- End: def
    
    def loadd(self, d):
        axes  = list(d.get('_axes', ()))
        shape = tuple(d.get('shape', ()))

        units = d.get('Units', None)
        if units is None:
            units = Units()

        dtype = d['dtype']
        self._dtype = dtype

        self.Units       = units
        self._axes       = axes
        self._flip       = list(d.get('_flip', ()))
        self._flatten    = list(d.get('_flatten', ()))
        self._fill_value = d.get('fill_value', None)

        self._shape = shape
        self._ndim  = len(shape)
        self._size  = long(reduce(operator_mul, shape, 1))

        cyclic = d.get('_cyclic', None)
        if cyclic:
            self._cyclic = cyclic.copy()
        else:
            self._cyclic = _empty_set

        filename = d.get('file', None)
        if filename is not None:
            filename = abspath(filename)

        base = d.get('base', None)
        if base is not None:
            base = abspath(base)

        # ------------------------------------------------------------
        # Initialise an empty partition array
        # ------------------------------------------------------------
        partition_matrix = PartitionMatrix(
            numpy_empty(d.get('_pmshape', ()), dtype=object),
            list(d.get('_pmaxes', ()))
            )
        pmndim = partition_matrix.ndim

        # ------------------------------------------------------------
        # Fill the partition array with partitions
        # ------------------------------------------------------------
        for attrs in d['Partitions']:

            # Find the position of this partition in the partition
            # matrix
            if 'index' in attrs:
                index = attrs['index']
                if len(index) == 1:
                    index = index[0]
                else:
                    index = tuple(index)
            else:
                index = (0,) * pmndim

            location = attrs.get('location', None)
            if location is not None:
                location = location[:]
            else:
                # Default location
                location = [[0, i] for i in shape]

            partition = Partition(
                location = location,
                axes     = attrs.get('axes', axes)[:],
                flip     = attrs.get('flip', [])[:],
                Units    = attrs.get('Units', units),
                part     = attrs.get('part', [])[:],
                flatten  = attrs.get('flatten', _empty_list)[:],
                )
            
            fmt = attrs.get('format', None)
            if fmt is None:
                # ----------------------------------------------------
                # Subarray is effectively a numpy array in memory
                # ----------------------------------------------------
                partition.subarray = attrs['subarray']

            else:
                # ----------------------------------------------------
                # Subarray is in a file on disk
                # ----------------------------------------------------
                partition.subarray = attrs['subarray']
                if fmt not in ('netCDF', 'UM'):
                    raise TypeError(
"Don't know how to load sub-array from {0!r} file format".format(fmt))

                # Set the 'subarray' attribute 
                kwargs = attrs['subarray'].copy()
                kwargs['shape'] = tuple(kwargs['shape'])
                
                kwargs['ndim'] = len(kwargs['shape'])
                kwargs['size'] = long(reduce(operator_mul, kwargs['shape'], 1))
                
                kwargs.setdefault('dtype', dtype)
                
                if 'file' in kwargs:
                    f = kwargs['file']
                    if f == '':
                        kwargs['file'] = filename
                    else:
                        if base is not None:
                            f = pathjoin(base, f)
                        kwargs['file'] = abspath(f)
                else:
                    kwargs['file'] = filename
                
                if fmt == 'netCDF':                
                    partition.subarray = NetCDFFileArray(**kwargs)
                elif fmt == 'UM':
                    partition.subarray = UMFileArray(**kwargs)
            #--- End: if

            # Put the partition into the partition array 
            partition_matrix[index] = partition
        #--- End: for

        # Save the partition array
        self.partitions = partition_matrix

        self.chunk()
        
        # ------------------------------------------------------------
        # Auxiliary mask
        # ------------------------------------------------------------
        auxiliary_mask = d.get('auxiliary_mask', None)
        if auxiliary_mask:
            self._auxiliary_mask = [m.copy() for m in auxiliary_mask]
        else:
            self._auxiliary_mask = None
    #--- End: def

    def chunk(self, chunksize=None):
        # Doesn't access the data array
        for partition in self.partitions.matrix.flat:
            
            n_chunks = int(partition.size*factor + 0.5)
            
            if n_chunks <= 1:
                continue
            
            extra_boundaries = []
            chunk_dims       = []
            
            for size, location, axis in izip(partition.shape, 
                                             partition.location,
                                             axes):
                if size == 1 or n_chunks <= 1:
                    continue
                
                if size <= n_chunks:
                    d[axis] = range(location[0]+1, location[1])

                    n_chunks = int(math_ceil(n_chunks/float(size)))

                else:
                    step = int(size/n_chunks)
                    d[axis] = range(location[0]+step, location[1], step)
                    
                    break
                #--- End: if
            #--- End: for                 
        #--- End: for
    #--- End: def

    def _binary_operation(self, other, method):
        '''

Implement binary arithmetic and comparison operations with the numpy
broadcasting rules.

It is called by the binary arithmetic and comparison
methods, such as `__sub__`, `__imul__`, `__rdiv__`, `__lt__`, etc.

.. seealso:: `_unary_operation`

:Parameters:

    other : object
        The object on the right hand side of the operator.

    method : str
        The binary arithmetic or comparison method name (such as
        ``'__imul__'`` or ``'__ge__'``).

:Returns:

    new : Data
        A new Data object, or if the operation was in place, the same
        Data object.

:Examples:

>>> d = cf.Data([0, 1, 2, 3])
>>> e = cf.Data([1, 1, 3, 4])

>>> f = d._binary_operation(e, '__add__')
>>> print f.array
[1 2 5 7]

>>> e = d._binary_operation(e, '__lt__')
>>> print e.array
[ True False  True  True]

>>> d._binary_operation(2, '__imul__')
>>> print d.array
[0 2 4 6]

'''
        inplace      = (method[2] == 'i')
        method_type  = method[-5:-2]

        # ------------------------------------------------------------
        # Ensure that other is an independent Data object
        # ------------------------------------------------------------    
        if getattr(other, '_NotImplemented_RHS_Data_op', False):
            # Make sure that 
            return NotImplemented

        elif not isinstance(other, self.__class__):
            other = type(self).asdata(other)

            if other._isdt and self.Units.isreftime:
                # Make sure that an array of date-time objects has the
                # right calendar.
                other.override_units(self.Units, i=True)
        #--- End: if

        data0 = self.copy()

        data0, other, new_Units = data0._combined_units(other, method, True)

        # ------------------------------------------------------------
        # Bring other into memory, if appropriate.
        # ------------------------------------------------------------
        other.to_memory()

        # ------------------------------------------------------------
        # Find which dimensions need to be broadcast in one or other
        # of the arrays.
        #
        # Method:
        #
        #   For each common dimension, the 'broadcast_indices' list
        #   will have a value of None if there is no broadcasting
        #   required (i.e. the two arrays have the same size along
        #   that dimension) or a value of slice(None) if broadcasting
        #   is required (i.e. the two arrays have the different sizes
        #   along that dimension and one of the sizes is 1).
        #   
        #   Example:
        #
        #     If c.shape is (7,1,6,1,5) and d.shape is (6,4,1) then
        #     broadcast_indices will be
        #     [None,slice(None),slice(None)].
        #   
        #     The indices to d which correspond to a partition of c,
        #     are the relevant subset of partition.indices updated
        #     with the non None elements of the broadcast_indices
        #     list.
        #   
        #     In this example, if a partition of c were to have a
        #     partition.indices value of (slice(0,3), slice(0,1),
        #     slice(2,4), slice(0,1), slice(0,5)), then the relevant
        #     subset of these is partition.indices[2:] and the
        #     corresponding indices to d are (slice(2,4), slice(None),
        #     slice(None))
        #
        # ------------------------------------------------------------
        data0_shape = data0._shape
        data1_shape = other._shape

        if data0_shape == data1_shape:
            # self and other have the same shapes
            broadcasting = False
            
            align_offset = 0
            ellipsis     = None

            new_shape = data0_shape
            new_ndim  = data0._ndim
            new_axes  = data0._axes
            new_size  = data0._size

        else:
            # self and other have different shapes
            broadcasting = True

            data0_ndim = data0._ndim
            data1_ndim = other._ndim

            align_offset = data0_ndim - data1_ndim
            if align_offset >= 0:
                # self has at least as many axes as other
                shape0 = data0_shape[align_offset:]
                shape1 = data1_shape

                new_shape = data0_shape[:align_offset]
                new_ndim  = data0_ndim
                new_axes  = data0._axes
            else:
                # other has more axes than self
                align_offset = -align_offset
                shape0       = data0_shape
                shape1       = data1_shape[align_offset:]

                new_shape = data1_shape[:align_offset]
                new_ndim  = data1_ndim
                if not data0_ndim:
                    new_axes = other._axes
                else:
                    new_axes = []
                    existing_axes = self._all_axis_names()
                    for n in new_shape:
                        axis = data0._new_axis_identifier(existing_axes)
                        existing_axes.append(axis)
                        new_axes.append(axis)
                    #--- End: for
                    new_axes += data0._axes
                #--- End: for
                     
                align_offset = 0
            #--- End: if

            broadcast_indices = []
            for a, b in izip(shape0, shape1):
                if a == b:
                    new_shape += (a,)
                    broadcast_indices.append(None)
                    continue
                
                # Still here?
                if a > 1 and b == 1:
                    new_shape += (a,)
                elif b > 1 and a == 1:
                    new_shape += (b,)
                else:
                    raise ValueError(
                        "Can't broadcast shape %s against shape %s" %
                        (data1_shape, data0_shape))
                
                broadcast_indices.append(slice(None))
            #--- End: for

            new_size = long(reduce(operator_mul, new_shape, 1))

            dummy_location = [None] * new_ndim
        #---End: if

        new_flip = []

#        if broadcasting:
#            max_size = 0
#            for partition in data0.partitions.matrix.flat:
#                indices0 = partition.indices
#                indices1 = tuple([
#                    (index if not broadcast_index else broadcast_index) 
#                    for index, broadcast_index in izip(
#                            indices0[align_offset:], broadcast_indices)
#                ])
#                indices1 = (Ellipsis,) + indices
#
#                shape0 = partition.shape
#                shape1 = [index.stop - index.start
#                          for index in parse_indices(other.shape, indices1)]
#                
#                broadcast_size = 1
#                for n0, n1 in izip_longest(shape0[::-1], shape1[::-1], fillvalue=1):
#                    if n0 > 1:
#                        broadcast_size *= n0
#                    else:
#                        broadcast_size *= n1
#                #--- End: for
#
#                if broadcast_size > max_size:
#                    max_size = broadcast_size
#            #--- End: for
#
#            chunksize = CHUNKSIZE()
#            ffff = max_size*(new_dtype.itemsize + 1)
#            if ffff > chunksize:
#                data0.chunk(chunksize*(chunksize/ffff))
#        #--- End: if


        # ------------------------------------------------------------
        # Create a Data object which just contains the metadata for
        # the result. If we're doing a binary arithmetic operation
        # then result will get filled with data and returned. If we're
        # an augmented arithmetic assignment then we'll update self
        # with this new metadata.
        # ------------------------------------------------------------

        #if new_shape != data0_shape:
        #    set_location_map = True
        #    new_size = self._size
        #    dummy_location   = [None] * new_ndim
        #else:
        #    set_location_map = False
        #    new_size = long(reduce(mul, new_shape, 1))

#        if not set_location_map:
#            new_size = long(reduce(mul, new_shape, 1))
#        else:
#            new_size = self._size

        result        = data0.copy()
        result._shape = new_shape
        result._ndim  = new_ndim 
        result._size  = new_size 
        result._axes  = new_axes
#        result._flip  = new_flip

        # Is the result an array of date-time objects?
        new_isdt = data0._isdt and new_Units.isreftime

        # ------------------------------------------------------------
        # Set the data type of the result
        # ------------------------------------------------------------
        if method_type in ('_eq', '_ne', '_lt', '_le', '_gt', '_ge'):
            new_dtype = numpy_dtype(bool)
        elif not new_isdt:
            if 'true' in method:
                new_dtype = numpy_dtype(float)
            elif not inplace:
                new_dtype = numpy_result_type(data0.dtype, other.dtype) 
            else:
                new_dtype = data0.dtype
        else:
            new_dtype = _dtype_object


        # ------------------------------------------------------------
        # Set flags to control whether or not the data of result and
        # self should be kept in memory
        # ------------------------------------------------------------
        keep_result_in_memory = result.fits_in_memory(new_dtype.itemsize+1)
        keep_self_in_memory   = data0.fits_in_memory(data0.dtype.itemsize+1)
        if not inplace:
            # When doing a binary arithmetic operation we need to
            # decide whether or not to keep self's data in memory
            revert_to_file = True
#            save_self      = not data0.fits_in_memory(data0.dtype.itemsize+1)
#            keep_self_in_memory = data0.fits_in_memory(data0.dtype.itemsize+1)
        else:
            # When doing an augmented arithmetic assignment we don't
            # need to keep self's original data in memory
            revert_to_file      = False
#            save_self           = False
#            keep_self_in_memory = True
        #--- End: if

#        dimensions     = self._axes
#        direction = self.direction
#        units     = self.Units

        pda_args = data0.pda_args(keep_in_memory=keep_self_in_memory,
                                  revert_to_file=revert_to_file)

        if data0._isdt:
            # If self is a datatime array then make sure that it gets
            # converted to a reference time array prior to the
            # operation
            pda_args['func']   = dt2rt
            pda_args['update'] = False
        #--- End: if                    

        original_numpy_seterr = numpy_seterr(**_seterr)

# Think about dtype, here.

            
        for partition_r, partition_s in izip(result.partitions.matrix.flat,
                                             data0.partitions.matrix.flat):

            indices = partition_s.indices

            array0  = partition_s.array(**pda_args)
#            print array0, type(array0)
            if broadcasting:
                indices = tuple([
                        (index if not broadcast_index else broadcast_index) 
                        for index, broadcast_index in izip(
                            indices[align_offset:], broadcast_indices)
                        ])
                indices = (Ellipsis,) + indices
            #--- End: if

#            if other._isdt:
#                # If other is a datatime array then make sure that it
#                # gets converted to a reference time array prior to
#                # the operation
#                array1 = other[indices].rtarray
#            else:

            array1 = other[indices].array #unsafe_array

            # UNRESOLVED ISSUE: array1 could be much larger than the chunk size.

            if not inplace:
                partition = partition_r
                partition.update_inplace_from(partition_s)
            else:
                partition = partition_s

            # --------------------------------------------------------
            # Do the binary operation on this partition's data
            # --------------------------------------------------------
            try:
#                print method, 'A', type(array0), array0.dtype, type(array1), array1.dtype
                array0 = getattr(array0, method)(array1)
#                print method, 'A', array0, type(array0), array0.dtype
            except FloatingPointError as error:
                # Floating point point errors have been trapped
                if _mask_fpe[0]:
                    # Redo the calculation ignoring the errors and
                    # then set invalid numbers to missing data
                    numpy_seterr(**_seterr_raise_to_ignore)
                    array0 = getattr(array0, method)(array1)
                    array0 = numpy_ma_masked_invalid(array0, copy=False)
                    numpy_seterr(**_seterr) 
                else:
                    # Raise the floating point error exception
                    raise FloatingPointError(error)
            except TypeError as error:
                if inplace:
                    raise TypeError(
"Incompatible result data type ({0!r}) for in-place {1!r} arithmetic".format(
numpy_result_type(array0.dtype, array1.dtype).name, array0.dtype.name))
                else:
                    raise TypeError(error)
            #--- End: try
           
            if new_isdt:
                # Convert result array to a date-time object array
                array0 = rt2dt(array0, data0.Units)
 
            if array0 is NotImplemented:
                array0 = numpy_zeros(partition.shape, dtype=bool)
                
            if not inplace:
                p_datatype = array0.dtype
                if new_dtype != p_datatype:
                    new_dtype = numpy_result_type(p_datatype, new_dtype)
                
            partition.subarray = array0
            partition.Units    = new_Units
            partition.axes     = new_axes
            partition.flip     = new_flip

            if broadcasting:
                partition.location = dummy_location
                partition.shape    = list(array0.shape)
            #--- End: if

            partition._original      = False
            partition._write_to_disk = False
            partition.close(keep_in_memory=keep_result_in_memory)

            if not inplace:
                partition_s.close() 
        #--- End: for

        # Reset numpy.seterr
        numpy_seterr(**original_numpy_seterr)
            
        if not inplace:
            result._isdt = new_isdt
            result.Units = new_Units
            result.dtype = new_dtype
            result._flip = new_flip

            if broadcasting:
                result.partitions.set_location_map(result._axes)

            return result
        else:
            # Update the metadata for the new master array in place
            data0._isdt  = new_isdt
            data0._shape = new_shape
            data0._ndim  = new_ndim
            data0._size  = new_size
            data0._axes  = new_axes
            data0._flip  = new_flip
            data0.Units  = new_Units
            data0.dtype  = new_dtype
   
            if broadcasting:
                data0.partitions.set_location_map(new_axes)

            self.__dict__ = data0.__dict__

            return self
    #--- End: def

    def _move_flip_to_partitions(self):
        # Doesn't access subarray
        for partition in self.partitions.matrix.flat:
            p_axes = partition.axes
            p_flip = partition.flip[:]
            for axis in flip:
                if axis in p_flip:
                    p_flip.remove(axis)
                elif axis in p_axes:
                    p_flip.append(axis)
            #--- End: for
            partition.flip = p_flip
        #--- End: for
    #--- End: def

    def _move_flatten_to_partitions(self):
        # Doesn't access subarray
        for partition in self.partitions.matrix.flat:
            partition.flatten = p_partition + flatten           
    #--- End: def

    def _unary_operation(self, operation):
        # Creates new subarray
        for partition in new.partitions.matrix.flat:
            array = partition.array(**pda_args)
            partition.subarray = getattr(operator, operation)(array)            
            partition.close()
        #--- End: for
    #--- End: def

    def _collapse(self, func, fpartial, ffinalise, axes=None, squeeze=False,
                  weights=None, mtol=1, units=None, i=False, **kwargs):
        '''

Collapse the data array in place.

:Parameters:

    func : function

    fpartial : function

    ffinalize : function

    axes : (sequence of) int, optional
        The axes to be collapsed. By default flattened input is
        used. Each axis is identified by its integer position. No axes
        are collapsed if *axes* is an empty sequence.

    squeeze : bool, optional
        If False then the axes which are collapsed are left in the
        result as axes with size 1. In this case the result will
        broadcast correctly against the original array. By default
        collapsed axes are removed.

    weights : *optional*

    kwargs : *optional*

    i : bool, optional
        If True then update the data array in place. By default a new
        data array is created.

:Returns:

    out : cf.Data

'''     
        if i:
            d = self
        else:
            d = self.copy()
 
        ndim = d._ndim
        self_axes  = d._axes
        self_shape = d._shape        

        original_self_axes = self_axes[:]

        if axes is None:
            # Collapse all axes
            axes = range(ndim)
            n_collapse_axes     = ndim
            n_non_collapse_axes = 0
            Nmax = d._size
        elif not axes and axes != 0:
            # Collapse no axes
            return d
        else:
            # Collapse some (maybe all) axes
            axes = d._parse_axes(axes, '_collapse')
            n_collapse_axes     = len(axes)
            n_non_collapse_axes = ndim - n_collapse_axes 
            Nmax = 1
            for i in axes:
                Nmax *= self_shape[i]
        #--- End: if
            
        #-------------------------------------------------------------
        # Parse the weights.
        #
        # * Change the keys from dimension names to the integer
        #   positions of the dimensions.
        #
        # * Make sure all non-null weights are Data objects.
        # ------------------------------------------------------------
        if weights is not None:
            if not isinstance(weights, dict):
                # If the shape of the weights is not the same as the
                # shape of the data array then the weights are assumed
                # to span the collapse axes in the order in which they
                # are given
                if numpy_shape(weights) == self_shape:
                    weights = {tuple(self_axes): weights}
                else:
                    weights = {tuple([self_axes[i] for i in axes]): weights}

            else:
                weights = weights.copy()
                weights_axes = set()
                for key, value in weights.items():
                    del weights[key]
                    key = d._parse_axes(key, 'asdasds12983487')
                    if weights_axes.intersection(key):
                        raise ValueError("Duplicate weights axis")
                    
                    weights_axes.update(key)
                    weights[tuple([self_axes[i] for i in key])] = value
                #--- End: for

                if not weights_axes.intersection(axes):
                    # Ignore all of the weights if none of them span
                    # any collapse axes
                    weights = {}
            #--- End: if

            for key, weight in weights.items():
                if weight is None or numpy_size(weight) == 1:
                    # Ignore undefined weights and size 1 weights
                    del weights[key]
                    continue

                weight_ndim = numpy_ndim(weight)
                if weight_ndim != len(key):
                    raise ValueError(
"Can't collapse: Incorrect number of weights axes (%d != %d)" %
(weight.ndim, len(key)))
                
                if weight_ndim > ndim:
                    raise ValueError(
"Can't collapse: Incorrect number of weights axes (%d > %d)" %
(weight.ndim, ndim))
                
                for n, axis in izip(numpy_shape(weight), key):
                    if n != self_shape[self_axes.index(axis)]:
                        raise ValueError(
                            "Can't collapse: Incorrect weights shape %r" %
                            numpy_shape(weight))
                #--- End :for

                # Convert weight to a data object, if necessary.
                weight = type(self).asdata(weight)

                if weight.dtype.char == 'S':
                    # Ignore string-valued weights
                    del weights[key]
                    continue

                weights[key] = weight
            #--- End: for
        #--- End: if

        if axes != range(n_non_collapse_axes, ndim):
            transpose_iaxes = [i for i in range(ndim) if i not in axes] + axes
            d.transpose(transpose_iaxes, i=True)
        #--- End: if

        if weights:
            # Optimize when weights span only non-partitioned axes
            # (do this before permuting the order of the weight
            # axes to be consistent with the order of the data
            # axes)
            weights = d._collapse_optimize_weights(weights)

            # Permute the order of the weight axes to be
            # consistent with the order of the data axes
            self_axes = d._axes              
            for key, w in weights.items():
                key1 = tuple([axis for axis in self_axes if axis in key])
                
                if key1 != key:
                    w = w.transpose([key.index(axis) for axis in key1])

                del weights[key]
                ikey = tuple([self_axes.index(axis) for axis in key1])

                weights[ikey] = w
            #--- End: for
                    
            # Add the weights to kwargs
            kwargs['weights'] = weights
        #--- End: if

        #-------------------------------------------------------------
        # Initialise the output data array
        #-------------------------------------------------------------
        new = d[(Ellipsis,) + (0,)*n_collapse_axes]
        
#        new._auxiliary_mask = None    # pending .....
        for partition in new.partitions.matrix.flat:
            del partition.subarray
    
        d.to_memory()

        keep_in_memory = new.fits_in_memory(new.dtype.itemsize+1)

        datatype = d.dtype
    
        if units is None:
            new_units = new.Units
        else:
            new_units = units

        p_axes  = new._axes[:n_non_collapse_axes]               
        p_units = new_units
    
        c_slice = (slice(None),) * n_collapse_axes

        for partition in new.partitions.matrix.flat:
            partition.open()
            partition.axes  = p_axes
            partition.flip  = []
            partition.part  = []
            partition.Units = p_units
    
            if squeeze:
                partition.location = partition.location[:n_non_collapse_axes]
                partition.shape    = partition.shape[:n_non_collapse_axes]

            indices = partition.indices[:n_non_collapse_axes] + c_slice

            partition.subarray = d._collapse_subspace(
                func, fpartial, ffinalise,
                indices, n_non_collapse_axes, n_collapse_axes,
                Nmax, mtol, **kwargs)

            p_datatype = partition.subarray.dtype
            if datatype != p_datatype:
                datatype = numpy_result_type(p_datatype, datatype)
                
            partition.close(keep_in_memory=keep_in_memory)
        #--- End: for

        new._all_axes = None
        new._flip     = []
        new.dtype     = datatype
        new.Units     = new_units

        if squeeze:
            new._axes  = p_axes
            new._ndim  = ndim - n_collapse_axes
            new._shape = new._shape[:new._ndim]
        else:
            new_axes = new._axes
            if new_axes != original_self_axes:
                iaxes = [new_axes.index(axis) for axis in original_self_axes]
                new.transpose(iaxes, i=True)
    
        # ------------------------------------------------------------
        # Update d in place
        # ------------------------------------------------------------
        d.__dict__ = new.__dict__
        
        # ------------------------------------------------------------
        # Return
        # ------------------------------------------------------------
        return d
    #--- End: def

    def _collapse_subspace(self, func, fpartial, ffinalise, indices, 
                           n_non_collapse_axes, n_collapse_axes,
                           Nmax, mtol, weights=None, **kwargs):
        '''

Collapse a subspace of a data array.

If set, *weights* and *kwargs* are passed to the function call. If
there is a *weights* keyword argument then this should either evaluate
to False or be a dictionary of weights for at least one of the data
dimensions.

:Parameters:

    func : function

    fpartial : function

    ffinalise : function

    indices: tuple
        The indices of the master array which would create the
        subspace.

    n_non_collapse_axes : int
        The number of data array axes which are not being
        collapsed. It is assumed that they are in the slowest moving
        positions.

    n_collapse_axes : int
        The number of data array axes which are being collapsed. It is
        assumed that they are in the fastest moving positions.

    weights : dict, optional

    kwargs : *optional*

:Returns:

    out : list

:Examples:

'''   
        ndim = self._ndim

        master_shape = self.shape

        data = self[indices]

#        if data._pmndim and data.fits_in_memory(data.dtype.itemsize+1):
#            data.varray

        # True iff at least two, but not all, axes are to be
        # collapsed.
        reshape = 1 < n_collapse_axes < ndim

        out = None
                    
        if n_collapse_axes == ndim:
            # All axes are to be collapsed
            kwargs.pop('axis', None)
        else:
            # At least one axis, but not all axes, are to be
            # collapsed. It is assumed that the collapse axes are in
            # the last (fastest varying) positions (-1, -2, ...). We
            # set kwargs['axis']=-1 (actually we use the +ve integer
            # equivalent of -1) if there is more then one collapse
            # axis because, in this case (i.e. reshape is True), we
            # will reshape everything.
            kwargs['axis'] = ndim - n_collapse_axes

        masked = False

        sub_samples  = 0

        pda_args = data.pda_args(revert_to_file=True) #, readonly=True)

        for i, partition in enumerate(data.partitions.matrix.flat):
            array = partition.array(**pda_args)
    
            p_masked = partition.masked

            if p_masked:
                masked = True
                if array.mask.all():
                    # The array is all missing data
                    partition.close()
                    continue
    
            # Still here? Then there are some non-missing sub-array
            # elements.
            if weights is not None:
                w = self._collapse_create_weights(array, partition.indices,
                                                  indices,
                                                  master_shape, weights,
                                                  n_non_collapse_axes,
                                                  n_collapse_axes)
                wmin = w.min()
                if wmin < 0:
                    raise ValueError("Can't collapse with negative weights")

                if wmin == 0:
                    # Mask the array where the weights are zero
                    array = numpy_ma_masked_where(w==0, array, copy=True)
                    if array.mask.all():
                        # The array is all missing data
                        partition.close()
                        continue
                #--- End: if
 
                kwargs['weights'] = w
            #--- End: if

            partition.close()

            if reshape:
                # At least two, but not all, axes are to be collapsed
                # => we need to reshape the array and the weights.
                shape = array.shape
                ndim = array.ndim
                new_shape  = shape[:n_non_collapse_axes]
                new_shape += (reduce(operator_mul, shape[n_non_collapse_axes:]),)
                array = numpy_reshape(array.copy(), new_shape)

                if weights is not None:
                    w = kwargs['weights']
                    if w.ndim < ndim:
                        # The weights span only collapse axes (as
                        # opposed to spanning all axes)
                        new_shape = (w.size,)
    
                    kwargs['weights'] = numpy_reshape(w, new_shape)
            #--- End: if  

            p_out = func(array, masked=p_masked, **kwargs)

            if out is None:
                if data.partitions.size == i + 1:
                    # There is exactly one partition so we are done
                    out = p_out
                    break

                out = fpartial(p_out)
            else:
                out = fpartial(out, p_out)
                   
            sub_samples += 1        
        #--- End: for

        if out is not None:
            # Finalise
            N, out = ffinalise(out, sub_samples)
            out = self._collapse_mask(out, masked, N, Nmax, mtol)
        else:
            # no data - retrun all masked
            out = numpy_ma_masked_all(data.shape[:n_non_collapse_axes],
                                      data.dtype)

        return out
    #--- End: def

    # ----------------------------------------------------------------
    # Attribute
    # ----------------------------------------------------------------
    @property
    def dtype(self):
            # Accesses subarray attributes
            for partition in flat:
                datatype = numpy_result_type(datatype, partition.subarray)
    #--- End: def

    def ismasked(self):
        # Accesses subarray READONLY
        for partition in self.partitions.matrix.flat:
            array = partition.array(**pda_args)

            if partition.masked:
                partition.close()
                return True
            
            partition.close()
        #--- End: for
    #--- End: def

    # ----------------------------------------------------------------
    # Attribute (read only)
    # ----------------------------------------------------------------
    @property
    def array(self):

        # Still here?
        array_out = numpy_empty(self._shape, dtype=out_data_type)
            # Accesses subarray READONLY
            for partition in partitions.matrix.flat:
                
                p_array = partition.array(**pda_args)

#                isdt.append(partition.isdt)

                if _dtarray:
                    if not partition.isdt:
                        # Convert the partition subarray to an array
                        # of date-time objects
                        p_array = rt2dt(p_array, units)
                elif partition.isdt:
                    # Convert the partition subarray to an array of
                    # reference time floats
                    p_array = dt2rt(p_array, None, units)

                # copy okect?
                
                if not masked and partition.masked:
                    array_out = array_out.view(numpy_ma_MaskedArray)
                    array_out.set_fill_value(self._fill_value)

                array_out[partition.indices] = p_array
                partition.close()
            #--- End: for
    #--- End: def

    @property
    def mask(self):
        # Accesses subarray READONLY. Creates new data object
        for partition in mask.partitions.matrix.flat:
            array = partition.array(**pda_args)
            
            if partition.masked:
                # Array is masked
                partition.subarray = array.mask.copy()                   
            else:
                # Array is not masked
                partition.subarray = numpy_zeros(array.shape, dtype=bool)

            partition.Units = _units_None

            partition.close()
        #--- End: for
    #--- End: def

    def all(self):
        pda_args = self.pda_args(revert_to_file=True) #, readonly=True)
        # Accesses the subarray READONLY              
        for partition in self.partitions.matrix.flat:
            array = partition.array(**pda_args)
            a = array.all()
            if not a and a is not numpy_ma_masked:
                partition.close()
                return False

            partition.close()
        #--- End: for

        return True
    #--- End: def

    def any(self):
        pda_args = self.pda_args(revert_to_file=True) #, readonly=True)
        # Accesses the subarray READONLY
        for partition in self.partitions.matrix.flat:
            array = partition.array(**pda_args)
            if array.any():
                partition.close()
                return True

            partition.close()
        #--- End: for
    #--- End: def

    @property
    def binary_mask(self):
        # Accesses the subarray READONLY. Creates a NEW DATA ARRAY
        for partition in binary_mask.partitions.matrix.flat:            
            array = partition.array(**pda_args)        

            if partition.masked:
                # data is masked
                partition.subarray = numpy_array(~array.mask, dtype='int32')
            else:
                # data is not masked
                partition.subarray = numpy_ones(array.shape, dtype='int32')

            partition.Units = _units_1
            partition.close()
        #--- End: for

        binary_mask.Units = _units_1

        return binary_mask
    #--- End: def

    def clip(self, a_min, a_max, units=None, i=False):
        # Changes the subarray
        for partition in d.partitions.matrix.flat:            
            array = partition.array(**pda_args)            
            array.clip(a_min, a_max, out=array)
            partition.close()
        #--- End: if

        return d
    #--- End: def


    def count(self):
        # Accesses the subarray READONLY
        for partition in self.partitions.matrix.flat:
            partition.open(pda_args)
            array = partition.array() #**pda_args)
            n += numpy_ma_count(array)
            partition.close()
        #--- End: for
    #--- End: def

    def asreftime(self, i=False):
        # Access the subarray. CHANGES IT
        pda_args = d.configuration(func=dt2rt, dtype=None)

        for partition in d.partitions.matrix.flat:
            partition.open(pda_args)
            array = partition.array() #**pda_args)
            p_units = partition.Units
            partition.Units = Units(p_units.units, p_units._utime.calendar)
            partition.close()
        #--- End: for
    #--- End: def

    def asdatetime(self, i=False):
        # Access the subarray. CHANGES IT
        pda_args = d.configuration(func=rt2dt, dtype=None)

        for partition in d.partitions.matrix.flat:
            partition.open(configure)
            array = partition.array( #**pda_args)
            p_units = partition.Units
            partition.Units = Units(p_units.units, p_units._utime.calendar)
            partition.close()
        #--- End: for
    #--- End: def
        
    def _YMDhms(self, attr):
        def func(array, units_in, dummy0, dummy1):
            if not self._isdt:
                array = rt2dt(array, units_in)

            return _array_getattr(array, attr)
        #--- End: def

        pda_args = new.configuration(func=func, dtype=None)

        # Accesses the subarray CHANGES IT
        for partition in new.partitions.matrix.flat:
            partition.open(pda_args)
            array = partition.array() #**pda_args)
            new_dtype = array.dtype
            partition.close()
        #--- End: for
    #--- End: def

    def unique(self):
        pda_args = self.pda_args(revert_to_file=True) #, readonly=True)
        # Access the data array READONLY. Creates new NUMPY ARRAY-> DATA object
        for partition in self.partitions.matrix.flat:
            array = partition.array(**pda_args)
            array = numpy_unique(array)

            if partition.masked:
                # Note that compressing a masked array may result in
                # an array with zero size
                array = array.compressed()

            size = array.size
            if size > 1:
                u.extend(array)
            elif size == 1:
                u.append(array.item())

            partition.close()
        #--- End: for

        u = numpy_unique(numpy_array(u, dtype=self.dtype))

        return type(self)(u, units=self.Units, dt=self._isdt)
    #--- End: def

    def equals(self, other, rtol=None, atol=None, ignore_fill_value=False,
        # Accesses the subarray READONLY (short circuit)
        for partition in self.partitions.matrix.flat:
            array0 = partition.array(**pda_args)
            array1 = other[partition.indices].varray
            partition.close()
            if not _numpy_allclose(array0, array1, rtol=rtol, atol=atol):
                if traceback:
                    print("%s: Different data array values" %
                          self.__class__.__name__)
                return False
        #--- End: for
    #--- End: def

    def expand_dims(self, position=0, i=False):
        # Doesn't acces the subarray
         for partition in d.partitions.matrix.flat:
            partition.location = partition.location[:]
            partition.shape    = partition.shape[:]
            
            partition.location.insert(position, location)
            partition.shape.insert(position, 1)
        #--- End: for

    def override_units(self, units, i=False):
        # MAYBE Accesses the subarray
        for partition in d.partitions.matrix.flat:
            p_units = partition.Units
            if not p_units or p_units == units:
                # No need to create the data array if the sub-array
                # units are the same as the master data array units or
                # the partition units are not set
                partition.Units = units
                continue
            #--- End: if

            partition.array(**pda_args)
            partition.Units = units
            partition.close()
        #--- End: for
    #--- End: def

    def override_calendar(self, calendar, i=False):
        # Doesn't access the subarray
        for partition in d.partitions.matrix.flat:            
            partition.Units = Units(partition.Units._units, calendar)
            partition.close()
        #--- End: for
    #--- End: def

    def to_disk(self):
        pda_args = self.pda_args(keep_in_memory=False)

        for partition in self.partitions.matrix.flat:
            if partition.in_memory:
                partition.array(**pda_args)
                partition.close()
    #--- End: def

    def to_memory(self, regardless=False):
        '''

Store each partition's data in memory in place if the master array is
smaller than the chunk size.

There is no change to partitions with data that are already in memory.

:Parameters:
    
    regardless : bool, optional
        If True then store all partitions' data in memory regardless
        of the size of the master array. By default only store all
        partitions' data in memory if the master array is smaller than
        the chunk size.

:Returns:

    None

:Examples:

>>> d.to_memory()
>>> d.to_memory(regardless=True)

'''
        if self.fits_in_memory(self.dtype.itemsize+1) or regardless:
            pda_args = self.pda_args(keep_in_memory=True)

            for partition in self.partitions.matrix.flat:
                if partition.on_disk:
                    partition.array(**pda_args)
                    partition.close()
    #--- End: def

    # ----------------------------------------------------------------

    # ----------------------------------------------------------------
    @property
    def in_memory(self):
        # Doesn't access the subarray
        for partition in self.partitions.matrix.flat:
            if not partition.in_memory:
                return False
        #--- End: for

        return True
    #--- End: def

    def mask_invalid(self, i=False):
        # Accesses the subarray. MAYBE CHANGES IT
        for partition in d.partitions.matrix.flat:
            array = partition.array(**pda_args)

            array = numpy_ma_masked_invalid(array, copy=False)
            array.shrink_mask()
            if array.mask is numpy_ma_nomask:
                array = array.data

            partition.subarray = array
            partition.close()
        #--- End: for
    #--- End: def

    def where(self, condition, x=None, y=None, i=False):
        '''

Set data array elements depending on a condition.

Elements are set differently depending on where the condition is True
or False. Two assignment values are given. From one of them, the data
array is set where the condition is True and where the condition is
False, the data array is set from the other.

Each assignment value may either contain a single datum, or is an
array-like object which is broadcastable shape of the data array.

**Missing data**

The treatment of missing data elements depends on the value of the
`hardmask` attribute. If it is True then masked elements will not
unmasked, otherwise masked elements may be set to any value.

In either case, unmasked elements may be set to any value (including
missing data).

Unmasked elements may be set to missing data by assignment to the
`cf.masked` constant or by assignment to a value which contains masked
elements.

.. seealso:: `cf.masked`, `hardmask`, `__setitem__`

:Parameters:

    condition : *optional*
        Define the condition which determines how to set the data
        array. The condition is any object which is broadcastable to
        the data array shape. The condition is True where the object
        broadcast to the data array shape is True. If *condition* is
        unset then it defaults to a condition of True everywhere.

    x, y :

        Specify the assignment value. Where the condition defined by
        the *arg* and *kwargs* arguments is True, set the data array
        from *x* and where the condition is False, set the data array
        from *y*. Arguments *x* and *y* are each one of:

          * ``None``. The appropriate elements of the data array are
            unchanged.
        ..

          * Any object which is broadcastable to the data array's
            shape. The appropriate elements of the data array are set
            to the corresponding values from the object broadcast to
            the data array shape.
                
    i : bool, optional
        If True then update the data array in place. By default a new
        data array is created.
 
:Returns:

    out : cf.Data

:Examples:

'''     
        def _slice_to_partition(data, indices):           
            ''' 
            
    Return a numpy array for the part of the `cf.Data` object which
    spans the given indices.

    :Parameters:
    
        data : cf.Data
    
        indices : tuple
           
    :Returns:
    
        out : numpy array

    '''
            indices2 = [(slice(0, 1) if n == 1 else i)
                        for n, i in zip(data.shape[::-1], indices[::-1])]

            return data[tuple(indices2)[::-1]].unsafe_array
        #--- End: def

        def _is_broadcastable(data0, data1, do_not_broadcast, is_scalar):
            '''

    Check that the input *data1* is broadcastable to *data0* and
    return *data1*, as a python scalar if possible.
    
    This function updates following lists: do_not_broadcast, is_scalar

    :Parameters:
    
        data : cf.Data

        do_not_broadcast : list

        is_scalar : list

    :Returns:
    
        out : cf.Data or scalar
    
    '''
            shape0 = data0._shape
            shape1 = data1._shape
            size1  = data1._size

            if shape1 == shape0:
                do_not_broadcast.append(True)
                is_scalar.append(False)

            elif size1 == 1:
                do_not_broadcast.append(False)
                is_scalar.append(True)
                # Replace data1 with its scalar value
                data1 = data1.datum(0)
                
            elif data1._ndim <= data0._ndim and size1 < data0._size:
                do_not_broadcast.append(False)
                is_scalar.append(False)
                for n, m in zip(shape1[::-1], shape0[::-1]):
                    if n != m and n != 1:
                        raise ValueError(
"Can't assign by where: Can't broadcast data with shape %s to shape %s" %
(shape1, shape0))
            else:
                raise ValueError(
"Can't assign by where: Can't broadcast data with shape %s to shape %s" %
(shape1, shape0))
            #--- End: if

            return data1
        #--- End: def

        if i:
            d = self
        else:
            d = self.copy()

        if x is None and y is None:
            return d

        do_not_broadcast = []
        is_scalar        = []

        # ------------------------------------------------------------
        # Make sure that the condition is a cf.Data object
        # ------------------------------------------------------------
        if not isinstance(condition, d.__class__):
            condition = type(d)(condition)

        # Check that the condition is broadcastable
        condition = _is_broadcastable(d, condition, do_not_broadcast, is_scalar)

        # ------------------------------------------------------------
        # Parse x and y so that each is one of a) None, b) a scalar or
        # c) a data array with the same shape as the master array
        # ------------------------------------------------------------
        xy = []
        for value in (x, y):
            if value is None or value is cf_masked:
                do_not_broadcast.append(False)
                is_scalar.append(True)

            else:
                # Make sure that the value is a cf.Data object and has
                # compatible units
                if not isinstance(value, d.__class__):
                    value = type(d)(value)
                else:
                    if value.Units.equivalent(d.Units):
                        if not value.Units.equals(d.Units):                    
                            value = value.copy()                    
                            value.Units = d.Units
                    elif value.Units:
                        raise ValueError(
                            "Some bad units %r, %r" % (d.Units, value.Units))
                #--- End: if

                # Check that the value is broadcastable
                value = _is_broadcastable(d, value, do_not_broadcast, is_scalar)
            #--- End: if

            xy.append(value)
        #--- End: for
        x, y = xy
        c_is_scalar, x_is_scalar, y_is_scalar = is_scalar

        #-------------------------------------------------------------
        # Shortcuts if the condition is a scalar
        #-------------------------------------------------------------
        if c_is_scalar :
            if condition:
                if x is not None:
                    d[...] = x

                return d
            else:
                if y is not None:
                    d[...] = y
                    
                return d
        #--- End: if

        # Still here?
        broadcast = not any(do_not_broadcast)

        hardmask = d._hardmask
        pda_args = d.pda_args() #readonly=True)
        
        for partition in d.partitions.matrix.flat:
            array = partition.array(**pda_args)

            # --------------------------------------------------------
            # Find the master array indices for this partition
            # --------------------------------------------------------
            shape = array.shape
            indices = partition.indices

            # --------------------------------------------------------
            # Find the condition for this partition
            # --------------------------------------------------------
            if c_is_scalar:
                c = condition
            else:
                c = _slice_to_partition(condition, indices)

            # --------------------------------------------------------
            # Find value to use where condition is True for this
            # partition
            # --------------------------------------------------------
            if x_is_scalar:  
                if x is None:
                    # Use d
                    T = array
                    T_masked = partition.masked
                else:
                    T = x
                    T_masked = x is cf_masked
            else:
                T = _slice_to_partition(x, indices)
                T_masked = numpy_ma_isMA(T) and numpy_ma_is_masked(T)
            #--- End: if

            # --------------------------------------------------------
            # Find value to use where condition is False for this
            # partition
            # --------------------------------------------------------
            if y_is_scalar:  
                if y is None:
                    # Use d
                    F = array
                    F_masked = partition.masked
                else:
                    F = y
                    F_masked = y is cf_masked
            else:
                F = _slice_to_partition(y, indices)
                F_masked = numpy_ma_isMA(F) and numpy_ma_is_masked(F)
            #--- End: if

            # --------------------------------------------------------
            # Make sure that at least one of the arrays is the same
            # shape as the partition
            # --------------------------------------------------------
            if broadcast:
                if x is cf_masked and y is cf_masked:
                    c = _broadcast(c, shape)
                else:
                    #max_sizes = max((numpy_size(c), numpy_size(T), numpy_size(F)))
                    c = _broadcast(c, shape)
                    #if numpy_size(c) == max_sizes:
                    #    c = _broadcast(c, shape)
                    #elif numpy_size(T) == max_sizes:
                    #    T = _broadcast(T, shape)
                    #else:
                    #    F = _broadcast(F, shape)
            #--- End: if

            # --------------------------------------------------------
            # Create a numpy array which takes vales from T where c
            # is True and from F where c is False
            # --------------------------------------------------------
            if T_masked or F_masked:
                # T and/or F have missing data
                new = numpy_ma_where(c, T, F)
                if partition.masked:
                    if hardmask:
                        # The original partition has missing data and
                        # a hardmask, so apply the original
                        # partition's mask to the new array.
                        new.mask |= array.mask
                    elif not numpy_ma_is_masked(new):
                        # The original partition has missing data and
                        # a softmask and the new array doesn't have
                        # missing data, so turn the new array into an
                        # unmasked array.
                        new = new.data[...]  

                elif not numpy_ma_is_masked(new):
                    # The original partition doesn't have missing data
                    # and neither does the new array
                    new = new.data[...]
           
            else:
                # Neither T nor F have missing data
                new = numpy_where(c, T, F)
                if partition.masked and hardmask:
                    # The original partition has missing data and a
                    # hardmask, so apply the original partition's mask
                    # to the new array.
                    new = numpy_ma_masked_where(array.mask, new, copy=False)
            #--- End: if
                            
            # --------------------------------------------------------
            # Replace the partition's subarray with the new numpy
            # array
            # --------------------------------------------------------
            partition.subarray = new

            partition.close()
        #--- End: for

        return d
    #--- End: def

   def squeeze
        # Doesn't access the subarray
        for partition in d.partitions.matrix.flat:
            p_location = partition.location[:]
            p_shape    = partition.shape[:]
            p_flip     = partition.flip[:]
          
            for i, axis in i_axis:
                p_location.pop(i)
                p_shape.pop(i)                
                if axis in p_flip:
                    p_flip.remove(axis)
            #--- End: for

            partition.location = p_location
            partition.shape    = p_shape            
            partition.flip     = p_flip
        #--- End: for
    #--- End: def

    def transpose(self, axes=None, i=False):
        # Doesn't access the subarray
        for partition in d.partitions.matrix.flat:
            location = partition.location
            shape    = partition.shape

            partition.location = [location[i] for i in iaxes]
            partition.shape    = [shape[i]    for i in iaxes]
        #--- End: for
    #--- End: de

    def func(self, f, units=None, out=False, i=False):
        datatype = d.dtype
        # Changes the subarray
        for partition in d.partitions.matrix.flat:
            partition.open(config)
            array = partition.array() #**pda_args)

            if out:
                f(array, out=array)
            else:
                array = f(array)

            p_datatype = array.dtype
            if datatype != p_datatype:
                datatype = numpy_result_type(p_datatype, datatype)
                
            partition.subarray = array

            if units is not None:
                partition.Units = units

            partition.close()
        #--- End: for            
    #--- End: def

def _overlapping_partitions(partitions, indices, axes, master_flip):
    '''

Return the nested list of (modified) partitions which overlap the
given indices to the master array.

:Parameters:

    partitions : cf.PartitionMatrix

    indices : tuple

    axes : sequence of str

    master_flip : list

:Returns:

    out : numpy array
        A numpy array of cf.Partition objects.

:Examples:

>>> type f.Data
<class 'cf.data.Data'>
>>> d._axes
['dim1', 'dim2', 'dim0']
>>> axis_to_position = {'dim0': 2, 'dim1': 0, 'dim2' : 1}
>>> indices = (slice(None), slice(5, 1, -2), [1,3,4,8])
>>> x = _overlapping_partitions(d.partitions, indices, axis_to_position, master_flip)

'''

    axis_to_position = {}
    for i, axis in enumerate(axes):
        axis_to_position[axis] = i

    if partitions.size == 1:
        partition = partitions.matrix.item()

        # Find out if this partition overlaps the original slice
        p_indices, shape = partition.overlaps(indices)

        if p_indices is None:
            # This partition is not in the slice out of bounds - raise
            # error?
            return
        
        # Still here? Create a new partition                      
        partition = partition.copy()
        partition.new_part(p_indices, axis_to_position, master_flip)
        partition.shape = shape
        
        new_partition_matrix      = numpy_empty(partitions.shape, dtype=object)
        new_partition_matrix[...] = partition
        
        return new_partition_matrix
    #--- End: if

    # Still here? Then there are 2 or more partitions.

    partitions_list        = []
    partitions_list_append = partitions_list.append

    flat_pm_indices        = []
    flat_pm_indices_append = flat_pm_indices.append

    partitions_flat = partitions.matrix.flat

    i = partitions_flat.index

    for partition in partitions_flat:
        # Find out if this partition overlaps the original slice
        p_indices, shape = partition.overlaps(indices)
        
        if p_indices is None:
            # This partition is not in the slice 
            i = partitions_flat.index
            continue

        # Still here? Then this partition overlaps the slice, so
        # create a new partition.
        partition = partition.copy()
        partition.new_part(p_indices, axis_to_position, master_flip)
        partition.shape = shape
        
        partitions_list_append(partition)

        flat_pm_indices_append(i)

        i = partitions_flat.index 
    #--- End: for

    new_shape = [len(set(s)) 
                 for s in numpy_unravel_index(flat_pm_indices, partitions.shape)]

    new_partition_matrix = numpy_empty((len(flat_pm_indices),), dtype=object)
    new_partition_matrix[...] = partitions_list
    new_partition_matrix.resize(new_shape)

    return new_partition_matrix
#--- End: def
