001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.text.Format;
018import java.util.Arrays;
019import java.util.List;
020
021import org.eclipse.january.DatasetException;
022import org.eclipse.january.IMonitor;
023import org.eclipse.january.MetadataException;
024import org.eclipse.january.metadata.ErrorMetadata;
025import org.eclipse.january.metadata.MetadataFactory;
026import org.eclipse.january.metadata.StatisticsMetadata;
027import org.eclipse.january.metadata.internal.ErrorMetadataImpl;
028import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
029import org.slf4j.Logger;
030import org.slf4j.LoggerFactory;
031
032/**
033 * Generic container class for data 
034 * <p>
035 * Each subclass has an array of primitive types, elements of this array are grouped or
036 * compounded to make items 
037 * <p>
038 * Data items can be boolean, integer, float, complex float, vector float, etc
039 */
040public abstract class AbstractDataset extends LazyDatasetBase implements Dataset {
041        // pin UID to base class
042        private static final long serialVersionUID = Dataset.serialVersionUID;
043
044        private static final Logger logger = LoggerFactory.getLogger(AbstractDataset.class);
045
046        protected int size; // number of items
047
048        transient protected AbstractDataset base; // is null when not a view
049        protected int[] stride; // can be null for row-major, contiguous datasets
050        protected int offset;
051
052        /**
053         * The data itself, held in a 1D array, but the object will wrap it to appear as possessing as many dimensions as
054         * wanted
055         */
056        protected Serializable odata = null;
057
058        /**
059         * Set aliased data as base data
060         */
061        abstract protected void setData();
062
063        /**
064         * Constructor required for serialisation.
065         */
066        public AbstractDataset() {
067        }
068
069        @Override
070        public synchronized Dataset synchronizedCopy() {
071                return clone();
072        }
073
074        @Override
075        public Class<?> getElementClass() {
076                return InterfaceUtils.getElementClass(getClass());
077        }
078
079        @Override
080        public int getDType() {
081                return DTypeUtils.getDType(getClass());
082        }
083
084        @Override
085        public int hashCode() {
086                return getStats().getHash(shape);
087        }
088
089        @Override
090        abstract public AbstractDataset clone();
091
092        protected Format stringFormat = null;
093
094        @Override
095        public void setStringFormat(Format format) {
096                stringFormat = format;
097        }
098
099        @Override
100        public Dataset copy(final int dtype) {
101                return copy(DTypeUtils.getInterface(dtype));
102        }
103
104        @Override
105        public <T extends Dataset> T copy(Class<T> clazz) {
106                return DatasetUtils.copy(clazz, this);
107        }
108
109        @Override
110        public Dataset cast(final int dtype) {
111                return cast(DTypeUtils.getInterface(dtype));
112        }
113
114        @SuppressWarnings("unchecked")
115        @Override
116        public <T extends Dataset> T cast(Class<T> clazz) {
117                if (clazz.isInstance(this)) {
118                        return (T) this;
119                }
120                return DatasetUtils.cast(clazz, this);
121        }
122
123        @SuppressWarnings("unchecked")
124        @Override
125        public <T extends Dataset> T cast(final int isize, Class<T> clazz, final boolean repeat) {
126                if (clazz.isInstance(this) && getElementsPerItem() == isize) {
127                        return (T) this;
128                }
129                return DatasetUtils.cast(isize, clazz, this, repeat);
130        }
131
132        @Override
133        public Dataset cast(final boolean repeat, final int dtype, final int isize) {
134                return cast(isize, DTypeUtils.getInterface(dtype), repeat);
135        }
136
137        @Override
138        abstract public AbstractDataset getView(boolean deepCopyMetadata);
139
140        /**
141         * Copy fields from original to view
142         * @param orig original
143         * @param view destination
144         * @param clone if true, then clone everything but bulk data
145         * @param cloneMetadata if true, clone metadata
146         */
147        protected static void copyToView(Dataset orig, AbstractDataset view, boolean clone, boolean cloneMetadata) {
148                view.name = orig.getName();
149                view.size = orig.getSize();
150                view.odata = orig.getBuffer();
151                view.offset = orig.getOffset();
152                AbstractDataset a = orig instanceof AbstractDataset ? (AbstractDataset) orig : null;
153                view.base = a == null ? null : a.base;
154
155                int[] s = a == null ? null : a.stride;
156                if (clone) {
157                        view.shape = orig.getShapeRef() == null ? null : orig.getShape();
158                        view.stride = s == null ? null : s.clone();
159                } else {
160                        view.shape = orig.getShapeRef();
161                        view.stride = s;
162                }
163
164                view.metadata = getMetadataMap(orig, cloneMetadata);
165                Class<? extends Dataset> oc = InterfaceUtils.findSubInterface(orig.getClass());
166                Class<? extends Dataset> vc = InterfaceUtils.findSubInterface(view.getClass());
167                if (!oc.equals(vc)) {
168                        view.setDirty();
169                }
170        }
171
172        @Override
173        public IntegerDataset getIndices() {
174                final IntegerDataset ret = DatasetUtils.indices(shape);
175                if (getName() != null) {
176                        ret.setName("Indices of " + getName());
177                }
178                return ret;
179        }
180
181        @Override
182        public Dataset getTransposedView(int... axes) {
183                axes = checkPermutatedAxes(shape, axes);
184
185                AbstractDataset t = getView(true);
186                if (axes == null || getRank() == 1)
187                        return t;
188
189                int rank = shape.length;
190                int[] tstride = new int[rank];
191                int[] toffset = new int[1];
192                int[] nshape = createStrides(new SliceND(shape), this, tstride, toffset);
193                int[] nstride = new int[rank];
194                for (int i = 0; i < rank; i++) {
195                        final int ax = axes[i];
196                        nstride[i] = tstride[ax];
197                        nshape[i] = shape[ax];
198                }
199                t.shape = nshape;
200                t.stride = nstride;
201                t.offset = toffset[0];
202                t.base = this;
203                t.setDirty();
204                t.transposeMetadata(axes);
205                return t;
206        }
207
208        @Override
209        public Dataset transpose(int... axes) {
210                Dataset t = getTransposedView(axes);
211                return t == null ? clone() : t.clone();
212        }
213
214        @Override
215        public Dataset swapAxes(int axis1, int axis2) {
216                int rank = shape.length;
217                axis1 = ShapeUtils.checkAxis(rank, axis1);
218                axis2 = ShapeUtils.checkAxis(rank, axis2);
219
220                if (rank == 1 || axis1 == axis2) {
221                        return this;
222                }
223
224                int[] axes = new int[rank];
225                for (int i = 0; i < rank; i++) {
226                        axes[i] = i;
227                }               
228
229                axes[axis1] = axis2;
230                axes[axis2] = axis1;
231                return getTransposedView(axes);
232        }
233
234        boolean isContiguous() {
235                if (stride == null)
236                        return true;
237
238                if (offset != 0)
239                        return false;
240
241                int s = getElementsPerItem();
242                for (int j = getRank() - 1; j >= 0; j--) {
243                        if (stride[j] != s) {
244                                return false;
245                        }
246                        s *= shape[j];
247                }
248
249                return true;
250        }
251
252        @Override
253        public Dataset flatten() {
254                if (!isContiguous()) { // need to make a copy if not contiguous
255                        return clone().flatten();
256                }
257                return reshape(size);
258        }
259
260        /**
261         * Fill dataset from object at depth dimension
262         * @param obj fill value
263         * @param depth dimension
264         * @param pos position
265         */
266        protected void fillData(Object obj, final int depth, final int[] pos) {
267                if (obj == null) {
268                        Class<?> c = InterfaceUtils.getElementClass(getClass());
269                        if (Float.class.equals(c)) {
270                                set(Float.NaN, pos);
271                        } else if (Float.class.equals(c)) {
272                                set(Double.NaN, pos);
273                        }
274                        return;
275                }
276
277                Class<?> clazz = obj.getClass();
278                if (obj instanceof List<?>) {
279                        List<?> jl = (List<?>) obj;
280                        int l = jl.size();
281                        for (int i = 0; i < l; i++) {
282                                Object lo = jl.get(i);
283                                fillData(lo, depth + 1, pos);
284                                pos[depth]++;
285                        }
286                        pos[depth] = 0;
287                } else if (clazz.isArray()) {
288                        int l = Array.getLength(obj);
289                        if (clazz.equals(odata.getClass())) {
290                                System.arraycopy(obj, 0, odata, get1DIndex(pos), l);
291                        } else if (clazz.getComponentType().isPrimitive()) {
292                                for (int i = 0; i < l; i++) {
293                                        set(Array.get(obj, i), pos);
294                                        pos[depth]++;
295                                }
296                                pos[depth] = 0;
297                        } else {
298                                for (int i = 0; i < l; i++) {
299                                        fillData(Array.get(obj, i), depth + 1, pos);
300                                        pos[depth]++;
301                                }
302                                pos[depth] = 0;
303                        }
304                } else if (obj instanceof IDataset) {
305                        boolean[] a = new boolean[shape.length];
306                        Arrays.fill(a, depth, a.length, true);
307                        setSlice(obj, getSliceIteratorFromAxes(pos, a));
308                } else {
309                        set(obj, pos);
310                }
311        }
312
313        @Override
314        public IndexIterator getIterator(final boolean withPosition) {
315                if (stride != null) {
316                        return base.getSize() == 1  ? (withPosition ? new PositionIterator(offset, shape) :
317                                new SingleItemIterator(offset, size)) : new StrideIterator(shape, stride, offset);
318                }
319                if (shape == null) {
320                        return new NullIterator();
321                }
322
323                return withPosition ? new ContiguousIteratorWithPosition(shape, size) : new ContiguousIterator(size);
324        }
325
326        @Override
327        public IndexIterator getIterator() {
328                return getIterator(false);
329        }
330
331        @Override
332        public PositionIterator getPositionIterator(final int... axes) {
333                return new PositionIterator(shape, axes);
334        }
335
336        @Override
337        public IndexIterator getSliceIterator(final int[] start, final int[] stop, final int[] step) {
338                return internalGetSliceIterator(new SliceND(shape, start, stop, step));
339        }
340
341        /**
342         * @param slice an n-D slice
343         * @return an slice iterator that operates like an IndexIterator
344         */
345        public IndexIterator getSliceIterator(SliceND slice) {
346                checkSliceND(slice);
347                return internalGetSliceIterator(slice);
348        }
349
350        /**
351         * @param slice an n-D slice
352         * @return an slice iterator that operates like an IndexIterator
353         */
354        protected IndexIterator internalGetSliceIterator(SliceND slice) {
355                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
356                        return new NullIterator(shape, slice.getShape());
357                }
358                if (stride != null) {
359                        return new StrideIterator(getElementsPerItem(), shape, stride, offset, slice);
360                }
361                return new SliceIterator(shape, size, slice);
362        }
363
364        @Override
365        public SliceIterator getSliceIteratorFromAxes(final int[] pos, boolean[] axes) {
366                int rank = shape.length;
367                int[] start;
368                int[] stop = new int[rank];
369                int[] step = new int[rank];
370
371                if (pos == null) {
372                        start = new int[rank];
373                } else if (pos.length == rank) {
374                        start = pos.clone();
375                } else {
376                        throw new IllegalArgumentException("pos array length is not equal to rank of dataset");
377                }
378                if (axes == null) {
379                        axes = new boolean[rank];
380                        Arrays.fill(axes, true);
381                } else if (axes.length != rank) {
382                        throw new IllegalArgumentException("axes array length is not equal to rank of dataset");
383                }
384
385                for (int i = 0; i < rank; i++) {
386                        if (axes[i]) {
387                                stop[i] = shape[i];
388                        } else {
389                                stop[i] = start[i] + 1;
390                        }
391                        step[i] = 1;
392                }
393                return (SliceIterator) getSliceIterator(start, stop, step);
394        }
395
396        @Override
397        public BooleanIterator getBooleanIterator(Dataset choice) {
398                return getBooleanIterator(choice, true);
399        }
400
401        @Override
402        public BooleanIterator getBooleanIterator(Dataset choice, boolean value) {
403                return BooleanIterator.createIterator(value, this, choice, this);
404        }
405
406        @Override
407        public Dataset getByBoolean(Dataset selection) {
408                checkCompatibility(selection);
409
410                final int length = ((Number) selection.sum()).intValue();
411                final int is = getElementsPerItem();
412                Dataset r = DatasetFactory.zeros(is, getClass(), length);
413                BooleanIterator biter = getBooleanIterator(selection);
414
415                int i = 0;
416                while (biter.hasNext()) {
417                        r.setObjectAbs(i, getObjectAbs(biter.index));
418                        i += is;
419                }
420                return r;
421        }
422
423        @Override
424        public Dataset getBy1DIndex(IntegerDataset index) {
425                final int is = getElementsPerItem();
426                final Dataset r = DatasetFactory.zeros(is, getClass(), index.getShapeRef());
427                final IntegerIterator iter = new IntegerIterator(index, size, is);
428
429                int i = 0;
430                while (iter.hasNext()) {
431                        r.setObjectAbs(i, getObjectAbs(iter.index));
432                        i += is;
433                }
434                return r;
435        }
436
437        @Override
438        public Dataset getByIndexes(final Object... indexes) {
439                final IntegersIterator iter = new IntegersIterator(shape, indexes);
440                final int is = getElementsPerItem();
441                final Dataset r = DatasetFactory.zeros(is, getClass(), iter.getShape());
442
443                final int[] pos = iter.getPos();
444                int i = 0;
445                while (iter.hasNext()) {
446                        r.setObjectAbs(i, getObject(pos));
447                        i += is;
448                }
449                return r;
450        }
451
452        @Override
453        public boolean hasFloatingPointElements() {
454                Class<?> cls = getElementClass();
455                return cls == Float.class || cls == Double.class;
456        }
457
458        @Override
459        public int getElementsPerItem() {
460                return InterfaceUtils.getElementsPerItem(getClass());
461        }
462
463        @Override
464        public int getItemBytes() {
465                return InterfaceUtils.getItemBytes(getElementsPerItem(), getClass());
466        }
467
468        @Override
469        public int getSize() {
470                return size;
471        }
472
473        @Override
474        public int[] getShape() {
475                // make a copy of the dimensions data, and put that out
476                if (shape == null) {
477                        logger.warn("Shape is null!!!");
478                        return new int[] {};
479                }
480                return shape.clone();
481        }
482
483        @Override
484        public int getRank() {
485                return shape == null ? 0 : shape.length;
486        }
487
488        @Override
489        public int getNbytes() {
490                return getSize() * getItemBytes();
491        }
492
493        /**
494         * Check for -1 placeholder in shape and replace if necessary
495         * @param shape to use
496         * @param size expected size
497         */
498        private void checkShape(int[] shape, int size) {
499                if (shape == null) {
500                        if (size == 0) {
501                                return;
502                        }
503                        logger.error("New shape must not be null for nonzero-sized dataset");
504                        throw new IllegalArgumentException("New shape must not be null for nonzero-sized dataset");
505                }
506                int rank = shape.length;
507                int found = -1;
508                int nsize = 1;
509                for (int i = 0; i < rank; i++) {
510                        int d = shape[i];
511                        if (d == -1) {
512                                if (found == -1) {
513                                        found = i;
514                                } else {
515                                        logger.error("Can only have one -1 placeholder in shape");
516                                        throw new IllegalArgumentException("Can only have one -1 placeholder in shape");
517                                }
518                        } else {
519                                nsize *= d;
520                        }
521                }
522                if (found >= 0) {
523                        shape[found] = size/nsize;
524                } else if (nsize != size && !(rank == 0 && size == 0)) {
525                        logger.error("New shape is not same size as old shape");
526                        throw new IllegalArgumentException("New size is not same as the old size. Old size is "+size+" new size is "+nsize+" and shape is "+Arrays.toString(shape));
527                }
528        }
529
530        @Override
531        public void setShape(final int... shape) {
532                int[] nshape = shape == null ? null : shape.clone();
533                checkShape(nshape, size);
534                if (Arrays.equals(this.shape, nshape)) {
535                        return;
536                }
537
538                if (stride != null) {
539                        // the only compatible shapes are ones where new dimensions are factors of old dimensions
540                        // or are combined adjacent old dimensions 
541                        int[] oshape = this.shape;
542                        int orank = oshape.length;
543                        int nrank = nshape.length;
544                        int diff = nrank - orank;
545                        int[] nstride = new int[nrank];
546                        boolean ones = true;
547                        // work forwards for broadcasting cases
548                        for (int i = 0, j = 0; i < orank || j < nrank;) {
549                                if (j >= diff && i < orank && j < nrank && oshape[i] == nshape[j]) {
550                                        nstride[j++] = stride[i++];
551                                } else if (j < nrank && nshape[j] == 1) {
552                                        nstride[j++] = 0;
553                                } else if (i < orank && oshape[i] == 1) {
554                                        i++;
555                                } else {
556                                        if (j < nrank)
557                                                j++;
558                                        if (i < orank)
559                                                i++;
560                                        ones = false;
561                                }
562                        }
563                        if (!ones) { // not just ones differ in shapes
564                                int[] ostride = stride;
565                                int ob = 0;
566                                int oe = 1;
567                                int nb = 0;
568                                int ne = 1;
569                                while (ob < orank && nb < nrank) {
570                                        int ol = oshape[ob];
571                                        int nl = nshape[nb];
572                                        
573                                        if (nl < ol) { // find group of shape dimensions that form common size
574                                                do { // case where new shape spreads single dimension over several dimensions
575                                                        if (ne == nrank) {
576                                                                break;
577                                                        }
578                                                        nl *= nshape[ne++];
579                                                } while (nl < ol);
580                                                if (nl != ol) {
581                                                        logger.error("Subshape is incompatible with single dimension");
582                                                        throw new IllegalArgumentException("Subshape is incompatible with single dimension");
583                                                }
584                                                int on = ne - 1;
585                                                while (nshape[on] == 1) {
586                                                        on--;
587                                                }
588
589                                                nstride[on] = ostride[ob];
590                                                for (int n = on - 1; n >= nb; n--) {
591                                                        if (nshape[n] == 1)
592                                                                continue;
593
594                                                        nstride[n] = nshape[on] * nstride[on];
595                                                        on = n;
596                                                }
597                                        } else if (ol < nl) {
598                                                do { // case where new shape combines several dimensions into one dimension
599                                                        if (oe == orank) {
600                                                                break;
601                                                        }
602                                                        ol *= oshape[oe++];
603                                                } while (ol < nl);
604                                                if (nl != ol) {
605                                                        logger.error("Single dimension is incompatible with subshape");
606                                                        throw new IllegalArgumentException("Single dimension is incompatible with subshape");
607                                                }
608
609                                                int oo = oe - 1;
610                                                while (oshape[oo] == 1) {
611                                                        oo--;
612                                                }
613                                                int os = ostride[oo];
614                                                for (int o = oo - 1; o >= ob; o--) {
615                                                        if (oshape[o] == 1)
616                                                                continue;
617                                                        if (ostride[o] != oshape[oo] * ostride[oo]) {
618                                                                logger.error("Subshape cannot be a non-contiguous view");
619                                                                throw new IllegalArgumentException("Subshape cannot be a non-contiguous view");
620                                                        }
621                                                        oo = o;
622                                                }
623                                                nstride[nb] = os;
624                                        } else {
625                                                nstride[nb] = ostride[ob];
626                                        }
627
628                                        ob = oe++;
629                                        nb = ne++;
630                                }
631                        }
632        
633                        stride = nstride;
634                }
635
636                setDirty();
637                if (this.shape != null && metadata != null) {
638                        reshapeMetadata(this.shape, nshape);
639                }
640                this.shape = nshape;
641        }
642
643        @Override
644        public int[] getShapeRef() {
645                return shape;
646        }
647
648        @Override
649        public int getOffset() {
650                return offset;
651        }
652
653        @Override
654        public int[] getStrides() {
655                return stride;
656        }
657
658        @Override
659        public Serializable getBuffer() {
660                return odata;
661        }
662
663        @Override
664        public void overrideInternal(Serializable buffer, int... shape) {
665                if (buffer != null) {
666                        odata = buffer;
667                        setData();
668                        setDirty();
669                }
670        
671                if (shape != null) {
672                        this.shape = shape.clone();
673                        size = ShapeUtils.calcSize(this.shape);
674                }
675        }
676
677        /**
678         * Create a stride array from dataset
679         * @param a dataset
680         * @param offset output offset
681         * @return new strides
682         */
683        public static int[] createStrides(Dataset a, final int[] offset) {
684                return createStrides(a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), offset);
685        }
686
687        /**
688         * Create a stride array from dataset
689         * @param isize item size
690         * @param shape to use
691         * @param oStride original stride
692         * @param oOffset original offset (only used if there is an original stride)
693         * @param offset output offset
694         * @return new strides
695         */
696        public static int[] createStrides(final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] offset) {
697                int rank = shape.length;
698                final int[] stride;
699                if (oStride == null) {
700                        offset[0] = 0;
701                        stride = new int[rank];
702                        int s = isize;
703                        for (int j = rank - 1; j >= 0; j--) {
704                                stride[j] = s;
705                                s *= shape[j];
706                        }
707                } else {
708                        offset[0] = oOffset;
709                        stride = oStride.clone();
710                }
711                return stride;
712        }
713
714        /**
715         * Create a stride array from slice information and a dataset
716         * @param slice an n-D slice
717         * @param a dataset
718         * @param stride output stride
719         * @param offset output offset
720         * @return new shape
721         */
722        public static int[] createStrides(final SliceND slice, final Dataset a, final int[] stride, final int[] offset) {
723                return createStrides(slice, a.getElementsPerItem(), a.getShapeRef(), a.getStrides(), a.getOffset(), stride, offset);
724        }
725
726        /**
727         * Create a stride array from slice and dataset information
728         * @param slice an n-D slice
729         * @param isize item size
730         * @param shape to use
731         * @param oStride original stride
732         * @param oOffset original offset (only used if there is an original stride)
733         * @param stride output stride
734         * @param offset output offset
735         * @return new shape
736         */
737        public static int[] createStrides(final SliceND slice, final int isize, final int[] shape, final int[] oStride, final int oOffset, final int[] stride, final int[] offset) {
738                int[] lstart = slice.getStart();
739                int[] lstep = slice.getStep();
740                int[] newShape = slice.getShape();
741                int rank = shape.length;
742
743                if (oStride == null) {
744                        int s = isize;
745                        offset[0] = 0;
746                        for (int j = rank - 1; j >= 0; j--) {
747                                stride[j] = s * lstep[j];
748                                offset[0] += s * lstart[j];
749                                s *= shape[j];
750                        }
751                } else {
752                        offset[0] = oOffset;
753                        for (int j = 0; j < rank; j++) {
754                                int s = oStride[j];
755                                stride[j] = lstep[j] * s;
756                                offset[0] += lstart[j] * s;
757                        }
758                }
759
760                return newShape;
761        }
762
763        @Override
764        public Dataset getBroadcastView(int... broadcastShape) {
765                AbstractDataset view = getView(true);
766                
767                if (!Arrays.equals(shape, broadcastShape)) {
768                        List<int[]> nShapes = BroadcastUtils.broadcastShapesToMax(broadcastShape, shape);
769                        view.setShape(nShapes.get(0));
770                        view.stride = BroadcastUtils.createBroadcastStrides(view, broadcastShape);
771                        view.base = this;
772                        view.shape = broadcastShape.clone();
773                        view.size = ShapeUtils.calcSize(broadcastShape);
774                        if (view.name == null || view.name.isEmpty()) {
775                                view.name = "Broadcast from " + Arrays.toString(shape);
776                        } else {
777                                view.name = "Broadcast of " + view.name + " from " + Arrays.toString(shape);
778                        }
779                }
780                return view;
781        }
782
783        @Override
784        public Dataset getSliceView(final int[] start, final int[] stop, final int[] step) {
785                return internalGetSliceView(new SliceND(shape, start, stop, step));
786        }
787
788        @Override
789        public Dataset getSliceView(Slice... slice) {
790                if (slice == null || slice.length == 0) {
791                        return getView(true);
792                }
793
794                return internalGetSliceView(new SliceND(shape, slice));
795        }
796
797        /**
798         * Get a slice of the dataset. The returned dataset is a view on a selection of items
799         * @param slice an n-D slice
800         * @return slice view
801         */
802        @Override
803        public Dataset getSliceView(SliceND slice) {
804                checkSliceND(slice);
805                return internalGetSliceView(slice);
806        }
807
808        private Dataset internalGetSliceView(SliceND slice) {
809                if (slice.isAll()) {
810                        return getView(true);
811                }
812
813                final int rank = shape.length;
814                int[] sStride = new int[rank];
815                int[] sOffset = new int[1];
816
817                int[] sShape = createStrides(slice, this, sStride, sOffset);
818        
819                AbstractDataset s = getView(false);
820                s.shape = sShape;
821                s.size = ShapeUtils.calcSize(sShape);
822                s.stride = sStride;
823                s.offset = sOffset[0];
824                s.base = this;
825
826                s.metadata = copyMetadata();
827                s.sliceMetadata(true, slice);
828
829                s.setDirty();
830                s.setName(name + BLOCK_OPEN + slice + BLOCK_CLOSE);
831
832                return s;
833        }
834
835        /**
836         * Get flattened view index of given position 
837         * @param pos
838         *            the integer array specifying the n-D position
839         * @return the index on the flattened dataset
840         */
841        private int getFlat1DIndex(final int[] pos) {
842                final int imax = pos.length;
843                if (imax == 0) {
844                        return 0;
845                }
846
847                return get1DIndexFromShape(pos);
848        }
849
850        /**
851         * @return index of first element
852         * @since 2.0
853         */
854        protected int getFirst1DIndex() {
855                if (shape == null) {
856                        throw new IllegalArgumentException("Cannot find an index from a null shape");
857                }
858                return stride == null ? 0 : offset;
859        }
860
861        @Override
862        public int get1DIndex(final int... n) {
863                if (n.length == 0 && shape.length == 0)
864                        return offset;
865
866                return stride == null ? get1DIndexFromShape(n) : get1DIndexFromStrides(n);
867        }
868
869        private static void throwAIOOBException(int i, int s, int d) {
870                throw new ArrayIndexOutOfBoundsException("Index (" + i + ") out of range [-" + s + "," + s
871                                + "] in dimension " + d);
872        }
873
874        /**
875         * @param i position in first dimension
876         * @return the index on the data array corresponding to that location
877         */
878        protected int get1DIndex(int i) {
879                if (shape == null) {
880                        throw new IllegalArgumentException("Cannot find an index from a null shape");
881                }
882                if (shape.length > 1) {
883                        logger.error("This dataset is not 1D but was addressed as such");
884                        throw new UnsupportedOperationException("This dataset is not 1D but was addressed as such");
885                }
886                if (i < 0) {
887                        i += shape[0];
888                }
889                if (i < 0 || i >= shape[0]) {
890                        throwAIOOBException(i, shape[0], 0);
891                }
892                return stride == null ? i : i*stride[0] + offset;
893        }
894
895        /**
896         * @param i position in first dimension
897         * @param j position in second dimension
898         * @return the index on the data array corresponding to that location
899         */
900        protected int get1DIndex(int i, int j) {
901                if (shape == null) {
902                        throw new IllegalArgumentException("Cannot find an index from a null shape");
903                }
904                if (shape.length != 2) {
905                        logger.error("This dataset is not 2D but was addressed as such");
906                        throw new UnsupportedOperationException("This dataset is not 2D but was addressed as such");
907                }
908                if (i < 0) {
909                        i += shape[0];
910                }
911                if (i < 0 || i >= shape[0]) {
912                        throwAIOOBException(i, shape[0], 0);
913                }
914                if (j < 0) {
915                        j += shape[1];
916                }
917                if (j < 0 || j >= shape[1]) {
918                        throwAIOOBException(i, shape[1], 1);
919                }
920                return stride == null ? i*shape[1] + j : i*stride[0] + j*stride[1] + offset;
921        }
922
923        protected int get1DIndexFromShape(final int[] n) {
924                return get1DIndexFromShape(shape, n);
925        }
926
927        protected static int get1DIndexFromShape(final int[] shape, final int[] n) {
928                if (shape == null) {
929                        throw new IllegalArgumentException("Cannot find an index from a null shape");
930                }
931                final int rank = shape.length;
932                if (rank != n.length) {
933                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
934                        logger.error(errMsg);
935                        throw new IllegalArgumentException(errMsg);
936                }
937                int index = 0;
938                for (int i = 0; i < rank; i++) {
939                        final int si = shape[i];
940                        int ni = n[i];
941                        if (ni < 0) {
942                                ni += si;
943                        }
944                        if (ni < 0 || ni >= si) {
945                                throwAIOOBException(ni, si, i);
946                        }
947                        index = index * si + ni;
948                }
949
950                return index;
951        }
952
953        private int get1DIndexFromStrides(final int[] n) {
954                return get1DIndexFromStrides(shape, stride, offset, n);
955        }
956
957        private static int get1DIndexFromStrides(final int[] shape, final int[] stride, final int offset, final int[] n) {
958                if (shape == null) {
959                        throw new IllegalArgumentException("Cannot find an index from a null shape");
960                }
961                final int rank = shape.length;
962                if (rank != n.length) {
963                        String errMsg = String.format("Number of position values must be equal to rank of %d", rank);
964                        logger.error(errMsg);
965                        throw new IllegalArgumentException(errMsg);
966                }
967                int index = offset;
968                for (int i = 0; i < rank; i++) {
969                        final int st = stride[i];
970                        if (st != 0) { // not broadcasted
971                                final int si = shape[i];
972                                int ni = n[i];
973                                if (ni < 0) {
974                                        ni += si;
975                                }
976                                if (ni < 0 || ni >= si) {
977                                        throwAIOOBException(ni, si, i);
978                                }
979                                index += st * ni;
980                        }
981                }
982                return index;
983        }
984
985        @Override
986        public int[] getNDPosition(final int n) {
987                if (isIndexInRange(n)) {
988                        throw new IllegalArgumentException("Index provided " + n
989                                        + "is larger then the size of the containing array");
990                }
991
992                return stride == null ? ShapeUtils.getNDPositionFromShape(n, shape) : getNDPositionFromStrides(n);
993        }
994
995        private boolean isIndexInRange(final int n) {
996                if (stride == null) {
997                        return n >= size;
998                }
999                return n >= getBufferLength();
1000        }
1001
1002        /**
1003         * @return entire buffer length
1004         */
1005        abstract protected int getBufferLength();
1006
1007        private int[] getNDPositionFromStrides(int n) {
1008                n -= offset;
1009                int rank = shape.length;
1010                if (rank == 1) {
1011                        return new int[] { n / stride[0] };
1012                }
1013
1014                int[] output = new int[rank];
1015                int i = 0;
1016                while (i != n) { // TODO find more efficient way than this exhaustive search
1017                        int j = rank - 1;
1018                        for (; j >= 0; j--) {
1019                                output[j]++;
1020                                i += stride[j];
1021                                if (output[j] >= shape[j]) {
1022                                        output[j] = 0;
1023                                        i -= shape[j] * stride[j];
1024                                } else {
1025                                        break;
1026                                }
1027                        }
1028                        if (j == -1) {
1029                                logger.error("Index was not found in this strided dataset");
1030                                throw new IllegalArgumentException("Index was not found in this strided dataset");
1031                        }
1032                }
1033
1034                return output;
1035        }
1036
1037        @Override
1038        public int checkAxis(int axis) {
1039                return ShapeUtils.checkAxis(shape.length, axis);
1040        }
1041
1042        @Deprecated
1043        protected static int checkAxis(int rank, int axis) {
1044                return ShapeUtils.checkAxis(rank, axis);
1045        }
1046
1047        protected static final char BLOCK_OPEN = '[';
1048        protected static final char BLOCK_CLOSE = ']';
1049
1050        @Override
1051        public String toString() {
1052                final int rank = shape == null ? 0 : shape.length;
1053                final StringBuilder out = new StringBuilder();
1054
1055                if (InterfaceUtils.isElemental(getClass())) {
1056                        out.append("Dataset ");
1057                } else {
1058                        out.append("Compound dataset (");
1059                        out.append(getElementsPerItem());
1060                        out.append(") ");
1061                }
1062
1063                if (name != null && name.length() > 0) {
1064                        out.append("'");
1065                        out.append(name);
1066                        out.append("' has shape ");
1067                } else {
1068                        out.append("shape is ");
1069                }
1070
1071                out.append(BLOCK_OPEN);
1072                if (rank > 0) {
1073                        out.append(shape[0]);
1074                }
1075                for (int i = 1; i < rank; i++) {
1076                        out.append(", " + shape[i]);
1077                }
1078                out.append(BLOCK_CLOSE);
1079                return out.toString();
1080        }
1081
1082        @Override
1083        public String toString(boolean showData) {
1084                if (!showData) {
1085                        return toString();
1086                }
1087
1088                if (size == 0) {
1089                        return "[]";
1090                }
1091
1092                final int rank = shape == null ? 0 : shape.length;
1093                final StringBuilder out = new StringBuilder();
1094
1095                if (rank > 0) {
1096                        int[] pos = new int[rank];
1097                        final StringBuilder lead = new StringBuilder();
1098                        printBlocks(out, lead, 0, pos);
1099                } else {
1100                        out.append(getString());
1101                }
1102                return out.toString();
1103        }
1104
1105        /**
1106         * Limit to strings output via the toString() method
1107         */
1108        private static int maxStringLength = 120;
1109
1110        /**
1111         * Set maximum line length for toString() method
1112         * @param maxLineLength limit on length of line
1113         */
1114        public static void setMaxLineLength(int maxLineLength) {
1115                maxStringLength = maxLineLength;
1116        }
1117
1118        /**
1119         * @return maximum line length for toString() method
1120         */
1121        public static int getMaxLineLength() {
1122                return maxStringLength;
1123        }
1124
1125        /**
1126         * Limit to number of sub-blocks output via the toString() method
1127         */
1128        private static final int MAX_SUBBLOCKS = 6;
1129
1130        private final static String SEPARATOR = ",";
1131        private final static String SPACE = " ";
1132        private final static String ELLIPSIS = "...";
1133        private final static String NEWLINE = "\n";
1134
1135        /**
1136         * Make a line of output for last dimension of dataset
1137         * @param end
1138         * @param start
1139         * @return line
1140         */
1141        private StringBuilder makeLine(final int end, final int[] start) {
1142                StringBuilder line = new StringBuilder();
1143                final int[] pos;
1144                if (end >= start.length) {
1145                        pos = Arrays.copyOf(start, end + 1);
1146                } else {
1147                        pos = start;
1148                }
1149                pos[end] = 0;
1150                line.append(BLOCK_OPEN);
1151                line.append(getString(pos));
1152
1153                final int length = shape[end];
1154
1155                // trim elements printed if length exceed estimate of maximum elements
1156                int excess = length - maxStringLength / 3; // space + number + separator
1157                int midIndex = -1;
1158                if (excess > 0) {
1159                        int index = (length - excess) / 2;
1160                        for (int y = 1; y < index; y++) {
1161                                line.append(SEPARATOR + SPACE);
1162                                pos[end] = y;
1163                                line.append(getString(pos));
1164                        }
1165                        midIndex = line.length() + 2;
1166                        index = (length + excess) / 2;
1167                        for (int y = index; y < length; y++) {
1168                                line.append(SEPARATOR + SPACE);
1169                                pos[end] = y;
1170                                line.append(getString(pos));
1171                        }
1172                } else {
1173                        for (int y = 1; y < length; y++) {
1174                                line.append(SEPARATOR + SPACE);
1175                                pos[end] = y;
1176                                line.append(getString(pos));
1177                        }
1178                }
1179                line.append(BLOCK_CLOSE);
1180
1181                // trim string down to limit
1182                int lineLength = line.length();
1183                excess = lineLength - maxStringLength - ELLIPSIS.length() - 1;
1184                if (excess > 0) {
1185                        int index = (lineLength - excess) / 2;
1186                        if (midIndex > 0 && index > midIndex) {
1187                                index = midIndex;
1188                        } else {
1189                                index = line.lastIndexOf(SEPARATOR, index) + 2;
1190                        }
1191                        StringBuilder out = new StringBuilder(line.subSequence(0, index));
1192                        out.append(ELLIPSIS + SEPARATOR);
1193                        index = (lineLength + excess) / 2;
1194                        if (midIndex > 0 && index <= midIndex) {
1195                                index = midIndex - 1;
1196                        } else {
1197                                index = line.indexOf(SEPARATOR, index) + 1;
1198                        }
1199                        out.append(line.subSequence(index, lineLength));
1200                        return out;
1201                } else if (midIndex > 0) { // add ellipsis
1202                        StringBuilder out = new StringBuilder(line.subSequence(0, midIndex));
1203                        out.append(ELLIPSIS + SEPARATOR + SPACE);
1204                        out.append(line.subSequence(midIndex, lineLength));
1205                        return out;
1206                }
1207
1208                return line;
1209        }
1210
1211        /**
1212         * recursive method to print blocks
1213         */
1214        private void printBlocks(final StringBuilder out, final StringBuilder lead, final int level, final int[] pos) {
1215                if (out.length() > 0) {
1216                        char last = out.charAt(out.length() - 1);
1217                        if (last != BLOCK_OPEN) {
1218                                out.append(lead);
1219                        }
1220                }
1221                final int end = getRank() - 1;
1222                if (level != end) {
1223                        out.append(BLOCK_OPEN);
1224                        int length = shape[level];
1225
1226                        // first sub-block
1227                        pos[level] = 0;
1228                        StringBuilder newlead = new StringBuilder(lead);
1229                        newlead.append(SPACE);
1230                        printBlocks(out, newlead, level + 1, pos);
1231                        if (length < 2) { // escape
1232                                out.append(BLOCK_CLOSE);
1233                                return;
1234                        }
1235
1236                        out.append(SEPARATOR + NEWLINE);
1237                        for (int i = level + 1; i < end; i++) {
1238                                out.append(NEWLINE);
1239                        }
1240
1241                        // middle sub-blocks
1242                        if (length < MAX_SUBBLOCKS) {
1243                                for (int x = 1; x < length - 1; x++) {
1244                                        pos[level] = x;
1245                                        printBlocks(out, newlead, level + 1, pos);
1246                                        if (end <= level + 1) {
1247                                                out.append(SEPARATOR + NEWLINE);
1248                                        } else {
1249                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1250                                        }
1251                                }
1252                        } else {
1253                                final int excess = length - MAX_SUBBLOCKS;
1254                                int xmax = (length - excess) / 2;
1255                                for (int x = 1; x < xmax; x++) {
1256                                        pos[level] = x;
1257                                        printBlocks(out, newlead, level + 1, pos);
1258                                        if (end <= level + 1) {
1259                                                out.append(SEPARATOR + NEWLINE);
1260                                        } else {
1261                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1262                                        }
1263                                }
1264                                out.append(newlead);
1265                                out.append(ELLIPSIS + SEPARATOR + NEWLINE);
1266                                xmax = (length + excess) / 2;
1267                                for (int x = xmax; x < length - 1; x++) {
1268                                        pos[level] = x;
1269                                        printBlocks(out, newlead, level + 1, pos);
1270                                        if (end <= level + 1) {
1271                                                out.append(SEPARATOR + NEWLINE);
1272                                        } else {
1273                                                out.append(SEPARATOR + NEWLINE + NEWLINE);
1274                                        }
1275                                }
1276                        }
1277
1278                        // last sub-block
1279                        pos[level] = length - 1;
1280                        printBlocks(out, newlead, level + 1, pos);
1281                        out.append(BLOCK_CLOSE);
1282                } else {
1283                        out.append(makeLine(end, pos));
1284                }
1285        }
1286
1287        @Override
1288        public Dataset squeezeEnds() {
1289                return squeeze(true);
1290        }
1291
1292        @Override
1293        public Dataset squeeze() {
1294                return squeeze(false);
1295        }
1296
1297        @Override
1298        public Dataset squeeze(boolean onlyFromEnds) {
1299                final int[] tshape = ShapeUtils.squeezeShape(shape, onlyFromEnds);
1300                final int[] oshape = shape;
1301                if (stride == null) {
1302                        shape = tshape;
1303                } else {
1304                        int rank = shape.length;
1305                        int trank = tshape.length;
1306                        if (trank < rank) {
1307                                int[] tstride = new int[tshape.length];
1308                                if (onlyFromEnds) {
1309                                        for (int i = 0; i < rank; i++) {
1310                                                if (shape[i] != 1) {
1311                                                        for (int k = 0; k < trank; k++) {
1312                                                                tstride[k] = stride[i++];
1313                                                        }
1314                                                        break;
1315                                                }
1316                                        }
1317                                } else {
1318                                        int t = 0;
1319                                        for (int i = 0; i < rank; i++) {
1320                                                if (shape[i] != 1) {
1321                                                        tstride[t++] = stride[i];
1322                                                }
1323                                        }
1324                                }
1325                                shape = tshape;
1326                                stride = tstride;
1327                        }
1328                }
1329
1330                setDirty();
1331                reshapeMetadata(oshape, shape);
1332                return this;
1333        }
1334
1335        @Override
1336        public boolean isCompatibleWith(final ILazyDataset g) {
1337                return ShapeUtils.areShapesCompatible(shape, g.getShape());
1338        }
1339
1340        @Override
1341        public void checkCompatibility(final ILazyDataset g) throws IllegalArgumentException {
1342                ShapeUtils.checkCompatibility(this, g);
1343        }
1344
1345        @Override
1346        public Dataset reshape(final int... shape) {
1347                Dataset a;
1348                try {
1349                        a = getView(true);
1350                        a.setShape(shape);
1351                } catch (IllegalArgumentException e) {
1352                        a = clone();
1353                        a.setShape(shape);
1354                }
1355                return a;
1356        }
1357
1358        /**
1359         * @param start begin
1360         * @param stop exclusive end
1361         * @param step number to skip
1362         * @return number of steps to take
1363         */
1364        protected static int calcSteps(final double start, final double stop, final double step) {
1365                return Math.max(0, (int) Math.ceil((stop - start) / step));
1366        }
1367
1368        @Override
1369        public boolean isComplex() {
1370                return false;
1371        }
1372
1373        @Override
1374        public Dataset getRealPart() {
1375                return this;
1376        }
1377
1378        @Override
1379        public Dataset getRealView() {
1380                return getView(true);
1381        }
1382
1383        @Override
1384        public Dataset getSlice(final int[] start, final int[] stop, final int[] step) {
1385                return internalGetSlice(new SliceND(shape, start, stop, step));
1386        }
1387
1388        @Override
1389        public Dataset getSlice(Slice... slice) {
1390                return internalGetSlice(new SliceND(shape, slice));
1391        }
1392
1393        @Override
1394        public Dataset getSlice(IMonitor monitor, Slice... slice) {
1395                return getSlice(slice);
1396        }
1397
1398        @Override
1399        public Dataset getSlice(IMonitor monitor, SliceND slice) {
1400                return getSlice(slice);
1401        }
1402
1403        @Override
1404        public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) {
1405                return getSlice(start, stop, step);
1406        }
1407
1408        /**
1409         * Get a slice of the dataset. The returned dataset is a copied selection of items
1410         * @param slice an n-D slice
1411         * @return The dataset of the sliced data
1412         */
1413        @Override
1414        public Dataset getSlice(final SliceND slice) {
1415                checkSliceND(slice);
1416                return internalGetSlice(slice);
1417        }
1418
1419        private Dataset internalGetSlice(final SliceND slice) {
1420                SliceIterator it = (SliceIterator) internalGetSliceIterator(slice);
1421                AbstractDataset s = getSlice(it);
1422                s.metadata = copyMetadata();
1423                s.setDirty();
1424                s.sliceMetadata(true, slice);
1425                return s;
1426        }
1427
1428        /**
1429         * Get a slice of the dataset. The returned dataset is a copied selection of items
1430         * 
1431         * @param iterator Slice iterator
1432         * @return The dataset of the sliced data
1433         */
1434        abstract public AbstractDataset getSlice(final SliceIterator iterator);
1435
1436        @Override
1437        public Dataset setSlice(final Object obj, final SliceND slice) {
1438                checkSliceND(slice);
1439                return internalSetSlice(obj, slice);
1440        }
1441
1442        private Dataset internalSetSlice(final Object obj, final SliceND slice) {
1443                Dataset ds;
1444                if (obj instanceof Dataset) {
1445                        ds = (Dataset) obj;
1446                } else if (obj instanceof IDataset) {
1447                        ds = DatasetUtils.convertToDataset((IDataset) obj);
1448                } else {
1449                        Class<? extends Dataset> dClass = getClass();
1450                        if (!BooleanDataset.class.equals(dClass)) {
1451                                dClass = InterfaceUtils.getLargestInterface(this);
1452                        }
1453                        ds = DatasetFactory.createFromObject(getElementsPerItem(), dClass, obj);
1454                }
1455
1456                return setSlicedView(getSliceView(slice), ds);
1457        }
1458
1459        @Override
1460        public Dataset setSlice(final Object obj, final int[] start, final int[] stop, final int[] step) {
1461                return internalSetSlice(obj, new SliceND(shape, start, stop, step));
1462        }
1463
1464        /**
1465         * Set a view of current dataset to given dataset with broadcasting
1466         * @param view destination
1467         * @param d source of data
1468         * @return this dataset
1469         */
1470        abstract Dataset setSlicedView(Dataset view, Dataset d);
1471
1472        @Override
1473        public Dataset setSlice(Object obj, Slice... slice) {
1474                if (slice == null || slice.length == 0) {
1475                        return internalSetSlice(obj, new SliceND(shape));
1476                }
1477                return internalSetSlice(obj, new SliceND(shape, slice));
1478        }
1479
1480        @Override
1481        public boolean all() {
1482                return Comparisons.allTrue(this);
1483        }
1484
1485        @Override
1486        public BooleanDataset all(final int axis) {
1487                return Comparisons.allTrue(this, axis);
1488        }
1489
1490        @Override
1491        public boolean any() {
1492                return Comparisons.anyTrue(this);
1493        }
1494
1495        @Override
1496        public BooleanDataset any(final int axis) {
1497                return Comparisons.anyTrue(this, axis);
1498        }
1499
1500        @Override
1501        public Dataset ifloorDivide(final Object o) {
1502                return idivide(o).ifloor();
1503        }
1504
1505        @Override
1506        public double residual(final Object o) {
1507                return residual(o, null, false);
1508        }
1509
1510        @Override
1511        public double residual(final Object o, boolean ignoreNaNs) {
1512                return residual(o, null, ignoreNaNs);
1513        }
1514
1515        /**
1516         * @return statistics
1517         * @since 2.0
1518         */
1519        @SuppressWarnings("unchecked")
1520        protected StatisticsMetadata<Number> getStats() {
1521                StatisticsMetadata<Number> md = getFirstMetadata(StatisticsMetadata.class);
1522                if (md == null || md.isDirty(this)) {
1523                        md = new StatisticsMetadataImpl<Number>();
1524                        md.initialize(this);
1525                        setMetadata(md);
1526                }
1527                return md;
1528        }
1529
1530        /**
1531         * @return statistics
1532         * @since 2.0
1533         */
1534        @SuppressWarnings("unchecked")
1535        protected StatisticsMetadata<String> getStringStats() {
1536                StatisticsMetadata<String> md = getFirstMetadata(StatisticsMetadata.class);
1537                if (md == null || md.isDirty(this)) {
1538                        md = new StatisticsMetadataImpl<String>();
1539                        md.initialize(this);
1540                        setMetadata(md);
1541                }
1542                return md;
1543        }
1544
1545        @Override
1546        public Number max(boolean... ignoreInvalids) {
1547                return getStats().getMaximum(ignoreInvalids);
1548        }
1549
1550        @Override
1551        public Dataset max(int axis, boolean... ignoreInvalids) {
1552                return getStats().getMaximum(axis, ignoreInvalids);
1553        }
1554
1555        @Override
1556        public Dataset max(int[] axes, boolean... ignoreInvalids) {
1557                return getStats().getMaximum(axes, ignoreInvalids);
1558        }
1559
1560        @Override
1561        public Number min(boolean... ignoreInvalids) {
1562                return getStats().getMinimum(ignoreInvalids);
1563        }
1564
1565        @Override
1566        public Dataset min(int axis, boolean... ignoreInvalids) {
1567                return getStats().getMinimum(axis, ignoreInvalids);
1568        }
1569
1570        @Override
1571        public Dataset min(int[] axes, boolean... ignoreInvalids) {
1572                return getStats().getMinimum(axes, ignoreInvalids);
1573        }
1574
1575        @Override
1576        public int argMax(boolean... ignoreInvalids) {
1577                return getFlat1DIndex(maxPos(ignoreInvalids));
1578        }
1579
1580        /**
1581         * @since 2.0
1582         */
1583        @Override
1584        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
1585                return (IntegerDataset) getStats().getArgMaximum(axis, ignoreInvalids);
1586        }
1587
1588        @Override
1589        public int argMin(boolean... ignoreInvalids) {
1590                return getFlat1DIndex(minPos(ignoreInvalids));
1591        }
1592
1593        /**
1594         * @since 2.0
1595         */
1596        @Override
1597        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
1598                return (IntegerDataset) getStats().getArgMinimum(axis, ignoreInvalids);
1599        }
1600
1601        @Override
1602        public Number peakToPeak(boolean... ignoreInvalids) {
1603                return InterfaceUtils.fromDoubleToBiggestNumber(getClass(), max(ignoreInvalids).doubleValue() - min(ignoreInvalids).doubleValue());
1604        }
1605
1606        @Override
1607        public Dataset peakToPeak(int axis,  boolean... ignoreInvalids) {
1608                return Maths.subtract(max(axis, ignoreInvalids), min(axis, ignoreInvalids));
1609        }
1610
1611        @Override
1612        public Dataset peakToPeak(int[] axes,  boolean... ignoreInvalids) {
1613                return Maths.subtract(max(axes, ignoreInvalids), min(axes, ignoreInvalids));
1614        }
1615
1616        @Override
1617        public long count(boolean... ignoreInvalids) {
1618                return getStats().getCount(ignoreInvalids);
1619        }
1620
1621        @Override
1622        public Dataset count(int axis, boolean... ignoreInvalids) {
1623                return getStats().getCount(axis, ignoreInvalids);
1624        }
1625
1626        @Override
1627        public Dataset count(int[] axes, boolean... ignoreInvalids) {
1628                return getStats().getCount(axes, ignoreInvalids);
1629        }
1630
1631        @Override
1632        public Object sum(boolean... ignoreInvalids) {
1633                return InterfaceUtils.toBiggestNumber(getClass(), getStats().getSum(ignoreInvalids));
1634        }
1635
1636        @Override
1637        public Dataset sum(int axis, boolean... ignoreInvalids) {
1638                return getStats().getSum(axis, ignoreInvalids);
1639        }
1640
1641        @Override
1642        public Dataset sum(int[] axes, boolean... ignoreInvalids) {
1643                return getStats().getSum(axes, ignoreInvalids);
1644        }
1645
1646        @Override
1647        public Object product(boolean... ignoreInvalids) {
1648                return Stats.product(this, ignoreInvalids);
1649        }
1650
1651        @Override
1652        public Dataset product(int axis, boolean... ignoreInvalids) {
1653                return Stats.product(this, axis, ignoreInvalids);
1654        }
1655
1656        @Override
1657        public Dataset product(int[] axes, boolean... ignoreInvalids) {
1658                return Stats.product(this, axes, ignoreInvalids);
1659        }
1660
1661        @Override
1662        public Object mean(boolean... ignoreInvalids) {
1663                return getStats().getMean(ignoreInvalids);
1664        }
1665
1666        @Override
1667        public Dataset mean(int axis, boolean... ignoreInvalids) {
1668                return getStats().getMean(axis, ignoreInvalids);
1669        }
1670
1671        @Override
1672        public Dataset mean(int[] axes, boolean... ignoreInvalids) {
1673                return getStats().getMean(axes, ignoreInvalids);
1674        }
1675
1676        @Override
1677        public double variance() {
1678                return variance(false);
1679        }
1680
1681        @Override
1682        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
1683                return getStats().getVariance(isWholePopulation, ignoreInvalids);
1684        }
1685
1686        @Override
1687        public Dataset variance(int axis) {
1688                return getStats().getVariance(axis, false);
1689        }
1690
1691        @Override
1692        public Dataset variance(int[] axes) {
1693                return getStats().getVariance(axes, false);
1694        }
1695
1696        @Override
1697        public Dataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1698                return getStats().getVariance(axis, isWholePopulation, ignoreInvalids);
1699        }
1700
1701        @Override
1702        public Dataset variance(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1703                return getStats().getVariance(axes, isWholePopulation, ignoreInvalids);
1704        }
1705
1706        @Override
1707        public double stdDeviation() {
1708                return Math.sqrt(variance());
1709        }
1710
1711        @Override
1712        public double stdDeviation(boolean isWholePopulation, boolean... ignoreInvalids) {
1713                return Math.sqrt(variance(isWholePopulation, ignoreInvalids));
1714        }
1715
1716        @Override
1717        public Dataset stdDeviation(int axis) {
1718                return Maths.sqrt(variance(axis, false));
1719        }
1720
1721        @Override
1722        public Dataset stdDeviation(int[] axes) {
1723                return Maths.sqrt(variance(axes, false));
1724        }
1725
1726        @Override
1727        public Dataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
1728                return Maths.sqrt(variance(axis, isWholePopulation, ignoreInvalids));
1729        }
1730
1731        @Override
1732        public Dataset stdDeviation(int[] axes, boolean isWholePopulation, boolean... ignoreInvalids) {
1733                return Maths.sqrt(variance(axes, isWholePopulation, ignoreInvalids));
1734        }
1735
1736        @Override
1737        public double rootMeanSquare(boolean... ignoreInvalids) {
1738                StatisticsMetadata<Number> stats = getStats();
1739                final double mean = stats.getMean(ignoreInvalids).doubleValue();
1740                final double var = stats.getVariance(true, ignoreInvalids);
1741                return Math.sqrt(var + mean * mean);
1742        }
1743
1744        @Override
1745        public Dataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
1746                StatisticsMetadata<Number> stats = getStats();
1747                Dataset v = stats.getVariance(axis, true, ignoreInvalids);
1748                Dataset m = stats.getMean(axis, ignoreInvalids);
1749                Dataset result = Maths.multiply(m, m);
1750                return Maths.sqrt(result.iadd(v));
1751        }
1752
1753        @Override
1754        public Dataset rootMeanSquare(int[] axes, boolean... ignoreInvalids) {
1755                StatisticsMetadata<Number> stats = getStats();
1756                Dataset v = stats.getVariance(axes, true, ignoreInvalids);
1757                Dataset m = stats.getMean(axes, ignoreInvalids);
1758                Dataset result = Maths.multiply(m, m);
1759                return Maths.sqrt(result.iadd(v));
1760        }
1761
1762        /**
1763         * Set item from compatible dataset in a direct and speedy way. Remember to setDirty afterwards.
1764         * 
1765         * @param dindex destination index
1766         * @param sindex source index
1767         * @param src
1768         *            is the source data buffer
1769         */
1770        protected abstract void setItemDirect(final int dindex, final int sindex, final Object src);
1771
1772        /**
1773         * @return error broadcasted to current shape
1774         */
1775        private Dataset getBroadcastedInternalError() {
1776                if (shape == null) {
1777                        throw new IllegalArgumentException("Cannot get error for null dataset");
1778                }
1779                ILazyDataset led = super.getErrors();
1780                if (led == null)
1781                        return null;
1782
1783                Dataset ed = null;
1784                try {
1785                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1786                } catch (DatasetException e) {
1787                        logger.error("Could not get data from lazy dataset", e);
1788                }
1789                if (led != ed) {
1790                        setErrors(ed); // set back
1791                }
1792
1793                return ed.getBroadcastView(shape);
1794        }
1795
1796        @Override
1797        public Dataset getErrors() {
1798                Dataset ed = getBroadcastedInternalError();
1799                if (ed == null)
1800                        return null;
1801
1802                return ed;
1803        }
1804
1805        @Override
1806        public double getError() {
1807                Dataset ed = getBroadcastedInternalError();
1808                if (ed == null)
1809                        return 0;
1810
1811                return ed.getDouble();
1812        }
1813
1814        @Override
1815        public double getError(final int i) {
1816                Dataset ed = getBroadcastedInternalError();
1817                if (ed == null)
1818                        return 0;
1819
1820                return ed.getDouble(i);
1821        }
1822
1823        @Override
1824        public double getError(final int i, final int j) {
1825                Dataset ed = getBroadcastedInternalError();
1826                if (ed == null)
1827                        return 0;
1828
1829                return ed.getDouble(i, j);
1830        }
1831
1832        @Override
1833        public double getError(int... pos) {
1834                Dataset ed = getBroadcastedInternalError();
1835                if (ed == null)
1836                        return 0;
1837
1838                return ed.getDouble(pos);
1839        }
1840
1841        @Override
1842        public double[] getErrorArray(final int i) {
1843                Dataset ed = getBroadcastedInternalError();
1844                if (ed == null)
1845                        return null;
1846
1847                return new double[] {getError(i)};
1848        }
1849
1850        @Override
1851        public double[] getErrorArray(final int i, final int j) {
1852                Dataset ed = getBroadcastedInternalError();
1853                if (ed == null)
1854                        return null;
1855
1856                return new double[] {getError(i, j)};
1857        }
1858
1859        @Override
1860        public double[] getErrorArray(int... pos) {
1861                Dataset ed = getBroadcastedInternalError();
1862                if (ed == null)
1863                        return null;
1864
1865                return new double[] {getError(pos)};
1866        }
1867
1868        protected Dataset getInternalSquaredError() {
1869                Dataset sed = getErrorBuffer().getBroadcastView(shape);
1870                return sed;
1871        }
1872
1873        @Override
1874        public Dataset getErrorBuffer() {
1875                ErrorMetadata emd = getErrorMetadata();
1876                if (emd == null)
1877                        return null;
1878
1879                if (!(emd instanceof ErrorMetadataImpl)) {
1880                        ILazyDataset led = emd.getError();
1881                        Dataset ed;
1882                        try {
1883                                ed = DatasetUtils.sliceAndConvertLazyDataset(led);
1884                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1885                                setMetadata(emd);
1886                                emd.setError(ed);
1887                        } catch (MetadataException me) {
1888                                logger.error("Could not create metadata", me);
1889                        } catch (DatasetException e) {
1890                                logger.error("Could not get data from lazy dataset", e);
1891                        }
1892                }
1893
1894                return ((ErrorMetadataImpl) emd).getSquaredError();
1895        }
1896
1897        /**
1898         * Set a copy of the buffer that backs the (squared) error data
1899         * @param buffer can be null, anything that can be used to create a DoubleDataset or CompoundDoubleDataset
1900         */
1901        @Override
1902        public void setErrorBuffer(Serializable buffer) {
1903                if (shape == null) {
1904                        throw new IllegalArgumentException("Cannot set error buffer for null dataset");
1905                }
1906                if (buffer == null) {
1907                        clearMetadata(ErrorMetadata.class);
1908                        return;
1909                }
1910
1911                IDataset d = (IDataset) createFromSerializable(buffer, false);
1912                ErrorMetadata emd = getErrorMetadata();
1913                if (!(emd instanceof ErrorMetadataImpl)) {
1914                        try {
1915                                emd = MetadataFactory.createMetadata(ErrorMetadata.class);
1916                                setMetadata(emd);
1917                        } catch (MetadataException me) {
1918                                logger.error("Could not create metadata", me);
1919                        }
1920                }
1921                ((ErrorMetadataImpl) emd).setSquaredError(d);
1922        }
1923}