001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.util.ArrayList;
018import java.util.Arrays;
019import java.util.Comparator;
020import java.util.List;
021
022import org.apache.commons.math3.util.MathArrays;
023import org.eclipse.january.DatasetException;
024import org.slf4j.Logger;
025import org.slf4j.LoggerFactory;
026
027/**
028 * Utilities for manipulating datasets
029 */
030@SuppressWarnings("unchecked")
031public class DatasetUtils {
032
033        /**
034         * Setup the logging facilities
035         */
036        transient protected static final Logger utilsLogger = LoggerFactory.getLogger(DatasetUtils.class);
037
038        /**
039         * Append copy of dataset with another dataset along n-th axis
040         *
041         * @param a
042         * @param b
043         * @param axis
044         *            number of axis (negative number counts from last)
045         * @return appended dataset
046         */
047        public static Dataset append(IDataset a, IDataset b, int axis) {
048                final int[] ashape = a.getShape();
049                final int rank = ashape.length;
050                final int[] bshape = b.getShape();
051                if (rank != bshape.length) {
052                        throw new IllegalArgumentException("Incompatible number of dimensions");
053                }
054                axis = ShapeUtils.checkAxis(rank, axis);
055
056                for (int i = 0; i < rank; i++) {
057                        if (i != axis && ashape[i] != bshape[i]) {
058                                throw new IllegalArgumentException("Incompatible dimensions");
059                        }
060                }
061                final int[] nshape = new int[rank];
062                for (int i = 0; i < rank; i++) {
063                        nshape[i] = ashape[i];
064                }
065                nshape[axis] += bshape[axis];
066                final Class<? extends Dataset> ot = InterfaceUtils.getInterface(b);
067                final Class<? extends Dataset> dt = InterfaceUtils.getInterface(a);
068                Dataset ds = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getBestInterface(ot, dt), nshape);
069                IndexIterator iter = ds.getIterator(true);
070                int[] pos = iter.getPos();
071                while (iter.hasNext()) {
072                        int d = ashape[axis];
073                        if (pos[axis] < d) {
074                                ds.setObjectAbs(iter.index, a.getObject(pos));
075                        } else {
076                                pos[axis] -= d;
077                                ds.setObjectAbs(iter.index, b.getObject(pos));
078                                pos[axis] += d;
079                        }
080                }
081
082                return ds;
083        }
084
085        /**
086         * Changes specific items of dataset by replacing them with other array
087         * @param a
088         * @param indices dataset interpreted as integers
089         * @param values
090         * @return changed dataset
091         */
092        public static <T extends Dataset> T put(final T a, final Dataset indices, Object values) {
093                IndexIterator it = indices.getIterator();
094                Dataset vd = DatasetFactory.createFromObject(values).flatten();
095                int vlen = vd.getSize();
096                int v = 0;
097                while (it.hasNext()) {
098                        if (v >= vlen) v -= vlen;
099
100                        a.setObjectAbs((int) indices.getElementLongAbs(it.index), vd.getObjectAbs(v++));
101                }
102                return a;
103        }
104
105        /**
106         * Changes specific items of dataset by replacing them with other array
107         * @param a
108         * @param indices
109         * @param values
110         * @return changed dataset
111         */
112        public static <T extends Dataset> T put(final T a, final int[] indices, Object values) {
113                int ilen = indices.length;
114                Dataset vd = DatasetFactory.createFromObject(values).flatten();
115                int vlen = vd.getSize();
116                for (int i = 0, v= 0; i < ilen; i++) {
117                        if (v >= vlen) v -= vlen;
118
119                        a.setObjectAbs(indices[i], vd.getObjectAbs(v++));
120                }
121                return a;
122        }
123
124        /**
125         * Take items from dataset along an axis
126         * @param indices dataset interpreted as integers
127         * @param axis if null, then use flattened view
128         * @return a sub-array
129         */
130        public static <T extends Dataset> T take(final T a, final Dataset indices, Integer axis) {
131                IntegerDataset indexes = (IntegerDataset) indices.flatten().cast(IntegerDataset.class);
132                return take(a, indexes.getData(), axis);
133        }
134
135        /**
136         * Take items from dataset along an axis
137         * @param indices
138         * @param axis if null, then use flattened view
139         * @return a sub-array
140         */
141        public static <T extends Dataset> T take(final T a, final int[] indices, Integer axis) {
142                if (indices == null || indices.length == 0) {
143                        utilsLogger.error("No indices given");
144                        throw new IllegalArgumentException("No indices given");
145                }
146                int[] ashape = a.getShape();
147                final int rank = ashape.length;
148                final Class<? extends Dataset> ac = a.getClass();
149                final int ilen = indices.length;
150                final int is = a.getElementsPerItem();
151
152                Dataset result;
153                if (axis == null) {
154                        ashape = new int[1];
155                        ashape[0] = ilen;
156                        result = DatasetFactory.zeros(is, ac, ashape);
157                        Serializable src = a.getBuffer();
158                        for (int i = 0; i < ilen; i++) {
159                                ((AbstractDataset) result).setItemDirect(i, indices[i], src);
160                        }
161                } else {
162                        axis = a.checkAxis(axis);
163                        ashape[axis] = ilen;
164                        result = DatasetFactory.zeros(is, ac, ashape);
165
166                        int[] dpos = new int[rank];
167                        int[] spos = new int[rank];
168                        boolean[] axes = new boolean[rank];
169                        Arrays.fill(axes, true);
170                        axes[axis] = false;
171                        Serializable src = a.getBuffer();
172                        for (int i = 0; i < ilen; i++) {
173                                spos[axis] = indices[i];
174                                dpos[axis] = i;
175                                SliceIterator siter = a.getSliceIteratorFromAxes(spos, axes);
176                                SliceIterator diter = result.getSliceIteratorFromAxes(dpos, axes);
177
178                                while (siter.hasNext() && diter.hasNext()) {
179                                        ((AbstractDataset) result).setItemDirect(diter.index, siter.index, src);
180                                }
181                        }
182                }
183                result.setDirty();
184                return (T) result;
185        }
186
187        /**
188         * Construct a dataset that contains the original dataset repeated the number
189         * of times in each axis given by corresponding entries in the reps array
190         *
191         * @param a
192         * @param reps
193         * @return tiled dataset
194         */
195        public static Dataset tile(final IDataset a, int... reps) {
196                int[] shape = a.getShape();
197                int rank = shape.length;
198                final int rlen = reps.length;
199
200                // expand shape
201                if (rank < rlen) {
202                        int[] newShape = new int[rlen];
203                        int extraRank = rlen - rank;
204                        for (int i = 0; i < extraRank; i++) {
205                                newShape[i] = 1;
206                        }
207                        for (int i = 0; i < rank; i++) {
208                                newShape[i+extraRank] = shape[i];
209                        }
210
211                        shape = newShape;
212                        rank = rlen;
213                } else if (rank > rlen) {
214                        int[] newReps = new int[rank];
215                        int extraRank = rank - rlen;
216                        for (int i = 0; i < extraRank; i++) {
217                                newReps[i] = 1;
218                        }
219                        for (int i = 0; i < rlen; i++) {
220                                newReps[i+extraRank] = reps[i];
221                        }
222                        reps = newReps;
223                }
224
225                // calculate new shape
226                int[] newShape = new int[rank];
227                for (int i = 0; i < rank; i++) {
228                        newShape[i] = shape[i]*reps[i];
229                }
230
231                Dataset tdata = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getInterfaceFromClass(a.getElementsPerItem(), a.getElementClass()), newShape);
232
233                // decide which way to put slices
234                boolean manyColumns;
235                if (rank == 1)
236                        manyColumns = true;
237                else
238                        manyColumns = shape[rank-1] > 64;
239
240                if (manyColumns) {
241                        // generate each start point and put a slice in
242                        IndexIterator iter = tdata.getSliceIterator(null, null, shape);
243                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(null, shape, null);
244                        final int[] pos = iter.getPos();
245                        while (iter.hasNext()) {
246                                siter.setStart(pos);
247                                tdata.setSlice(a, siter);
248                        }
249
250                } else {
251                        // for each value, set slice given by repeats
252                        final int[] skip = new int[rank];
253                        for (int i = 0; i < rank; i++) {
254                                if (reps[i] == 1) {
255                                        skip[i] = newShape[i];
256                                } else {
257                                        skip[i] = shape[i];
258                                }
259                        }
260
261                        Dataset aa = convertToDataset(a);
262                        IndexIterator ita = aa.getIterator(true);
263                        final int[] pos = ita.getPos();
264
265                        final int[] sstart = new int[rank];
266                        final int extra = rank - pos.length;
267                        for (int i = 0; i < extra; i++) {
268                                sstart[i] = 0;
269                        }
270                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(sstart, null, skip);
271                        while (ita.hasNext()) {
272                                for (int i = 0; i < pos.length; i++) {
273                                        sstart[i + extra] = pos[i];
274                                }
275                                siter.setStart(sstart);
276                                tdata.setSlice(aa.getObjectAbs(ita.index), siter);
277                        }
278                }
279
280                return tdata;
281        }
282
283        /**
284         * Permute copy of dataset's axes so that given order is old order:
285         * <pre>{@literal 
286         *  axisPerm = (p(0), p(1),...) => newdata(n(0), n(1),...) = olddata(o(0), o(1), ...)
287         *  such that n(i) = o(p(i)) for all i
288         * }</pre>
289         * I.e. for a 3D dataset (1,0,2) implies the new dataset has its 1st dimension
290         * running along the old dataset's 2nd dimension and the new 2nd is the old 1st.
291         * The 3rd dimension is left unchanged.
292         *
293         * @param a
294         * @param axes if null or zero length then axes order reversed
295         * @return remapped copy of data
296         */
297        public static Dataset transpose(final IDataset a, int... axes) {
298                return convertToDataset(a).transpose(axes);
299        }
300
301        /**
302         * Swap two axes in dataset
303         * @param a
304         * @param axis1
305         * @param axis2
306         * @return swapped dataset
307         */
308        public static Dataset swapAxes(final IDataset a, int axis1, int axis2) {
309                return convertToDataset(a).swapAxes(axis1, axis2);
310        }
311
312        /**
313         * @param a
314         * @return sorted flattened copy of dataset
315         */
316        public static <T extends Dataset> T sort(final T a) {
317                return sort(a, (Integer) null);
318        }
319
320        /**
321         * @param a
322         * @param axis to sort along, if null then dataset is first flattened
323         * @return dataset sorted along axis
324         */
325        public static <T extends Dataset> T sort(final T a, final Integer axis) {
326                Dataset s = a.clone();
327                return (T) s.sort(axis);
328        }
329
330        /**
331         * Sort in place given dataset and reorder ancillary datasets too
332         * @param a dataset to be sorted
333         * @param b ancillary datasets
334         */
335        public static void sort(Dataset a, Dataset... b) {
336                if (!DTypeUtils.isDTypeNumerical(a.getDType())) {
337                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
338                }
339
340                // gather all datasets as double dataset copies
341                DoubleDataset s = copy(DoubleDataset.class, a);
342                int l = b == null ? 0 : b.length;
343                DoubleDataset[] t = new DoubleDataset[l];
344                int n = 0;
345                for (int i = 0; i < l; i++) {
346                        if (b[i] != null) {
347                                if (!DTypeUtils.isDTypeNumerical(b[i].getDType())) {
348                                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
349                                }
350                                t[i] = copy(DoubleDataset.class, b[i]);
351                                n++;
352                        }
353                }
354
355                double[][] y = new double[n][];
356                for (int i = 0, j = 0; i < l; i++) {
357                        if (t[i] != null) {
358                                y[j++] = t[i].getData();
359                        }
360                }
361
362                MathArrays.sortInPlace(s.getData(), y);
363
364                a.setSlice(s);
365                for (int i = 0; i < l; i++) {
366                        if (b[i] != null) {
367                                b[i].setSlice(t[i]);
368                        }
369                }
370        }
371
372        /**
373         * Indirectly sort along given axis
374         * @param a dataset whose indexes will be sorted
375         * @param axis to sort along, if null then dataset is first flattened
376         * @return indexes
377         * @since 2.1
378         */
379        public static IntegerDataset indexSort(Dataset a, Integer axis) {
380                if (axis == null) {
381                        int size = a.getSize();
382                        Integer[] index = new Integer[size];
383                        for (int i = 0; i < size; i++) {
384                                index[i] = i;
385                        }
386                        final Dataset f = a.flatten(); // is this correct for views??? Check with NumPy
387                        Comparator<Integer> cmp = new Comparator<Integer>() {
388
389                                @Override
390                                public int compare(Integer o1, Integer o2) {
391
392                                        return Double.compare(f.getElementDoubleAbs(o1), f.getElementDoubleAbs(o2));
393                                }
394                        };
395                        Arrays.sort(index, cmp);
396                        return DatasetFactory.createFromObject(IntegerDataset.class, index);
397                }
398
399                axis = a.checkAxis(axis);
400                final int[] shape = a.getShapeRef();
401                IntegerDataset id = DatasetFactory.zeros(IntegerDataset.class, shape);
402                int size = shape[axis];
403                Integer[] index = new Integer[size];
404
405                int[] dShape = new int[shape.length];
406                Arrays.fill(dShape, 1);
407                dShape[axis] = size;
408                final DoubleDataset dd = DatasetFactory.zeros(DoubleDataset.class, dShape);
409                final Comparator<Integer> cmp = new Comparator<Integer>() {
410                        @Override
411                        public int compare(Integer o1, Integer o2) {
412
413                                return Double.compare(dd.getElementDoubleAbs(o1), dd.getElementDoubleAbs(o2));
414                        }
415                };
416
417                SliceND ds = new SliceND(dShape);
418                SliceNDIterator it = new SliceNDIterator(new SliceND(shape), axis);
419                int[] pos = it.getPos();
420                int[] ipos = pos.clone();
421                while (it.hasNext()) {
422                        dd.setSlice(a.getSliceView(it.getCurrentSlice()), ds);
423                        for (int i = 0; i < size; i++) {
424                                index[i] = i;
425                        }
426                        Arrays.sort(index, cmp);
427
428                        System.arraycopy(pos, 0, ipos, 0, pos.length);
429                        for (int i = 0; i < size; i++) {
430                                ipos[axis] = i;
431                                id.set(index[i], ipos);
432                        }
433                }
434
435                return id;
436        }
437
438        /**
439         * Concatenate the set of datasets along given axis
440         * @param as
441         * @param axis
442         * @return concatenated dataset
443         */
444        public static Dataset concatenate(final IDataset[] as, int axis) {
445                if (as == null || as.length == 0) {
446                        utilsLogger.error("No datasets given");
447                        throw new IllegalArgumentException("No datasets given");
448                }
449                IDataset a = as[0];
450                if (as.length == 1) {
451                        return convertToDataset(a.clone());
452                }
453
454                int[] ashape = a.getShape();
455                axis = ShapeUtils.checkAxis(ashape.length, axis);
456                Class<? extends Dataset> ac = InterfaceUtils.getInterface(a);
457                int anum = as.length;
458                int isize = a.getElementsPerItem();
459
460                int i = 1;
461                for (; i < anum; i++) {
462                        if (!ac.equals(InterfaceUtils.getInterface(as[i]))) {
463                                utilsLogger.error("Datasets are not of same type");
464                                break;
465                        }
466                        if (!ShapeUtils.areShapesCompatible(ashape, as[i].getShape(), axis)) {
467                                utilsLogger.error("Datasets' shapes are not equal");
468                                break;
469                        }
470                        final int is = as[i].getElementsPerItem();
471                        if (isize < is)
472                                isize = is;
473                }
474                if (i < anum) {
475                        utilsLogger.error("Dataset are not compatible");
476                        throw new IllegalArgumentException("Datasets are not compatible");
477                }
478
479                for (i = 1; i < anum; i++) {
480                        ashape[axis] += as[i].getShape()[axis];
481                }
482
483                Dataset result = DatasetFactory.zeros(isize, ac, ashape);
484
485                int[] start = new int[ashape.length];
486                int[] stop = ashape;
487                stop[axis] = 0;
488                for (i = 0; i < anum; i++) {
489                        IDataset b = as[i];
490                        int[] bshape = b.getShape();
491                        stop[axis] += bshape[axis];
492                        result.setSlice(b, start, stop, null);
493                        start[axis] += bshape[axis];
494                }
495
496                return result;
497        }
498
499        /**
500         * Split a dataset into equal sections along given axis
501         * @param a
502         * @param sections
503         * @param axis
504         * @param checkEqual makes sure the division is into equal parts
505         * @return list of split datasets
506         */
507        public static List<Dataset> split(final Dataset a, int sections, int axis, final boolean checkEqual) {
508                int[] ashape = a.getShapeRef();
509                axis = a.checkAxis(axis);
510                int imax = ashape[axis];
511                if (checkEqual && (imax%sections) != 0) {
512                        utilsLogger.error("Number of sections does not divide axis into equal parts");
513                        throw new IllegalArgumentException("Number of sections does not divide axis into equal parts");
514                }
515                int n = (imax + sections - 1) / sections;
516                int[] indices = new int[sections-1];
517                for (int i = 1; i < sections; i++)
518                        indices[i-1] = n*i;
519                return split(a, indices, axis);
520        }
521
522        /**
523         * Split a dataset into parts along given axis
524         * @param a
525         * @param indices
526         * @param axis
527         * @return list of split datasets
528         */
529        public static List<Dataset> split(final Dataset a, int[] indices, int axis) {
530                final int[] ashape = a.getShapeRef();
531                axis = a.checkAxis(axis);
532                final int rank = ashape.length;
533                final int imax = ashape[axis];
534
535                final List<Dataset> result = new ArrayList<Dataset>();
536
537                final int[] nshape = ashape.clone();
538                final int is = a.getElementsPerItem();
539
540                int oind = 0;
541                final int[] start = new int[rank];
542                final int[] stop = new int[rank];
543                final int[] step = new int[rank];
544                for (int i = 0; i < rank; i++) {
545                        start[i] = 0;
546                        stop[i] = ashape[i];
547                        step[i] = 1;
548                }
549                for (int ind : indices) {
550                        if (ind > imax) {
551                                result.add(DatasetFactory.zeros(is, a.getClass(), 0));
552                        } else {
553                                nshape[axis] = ind - oind;
554                                start[axis] = oind;
555                                stop[axis] = ind;
556                                Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
557                                IndexIterator iter = a.getSliceIterator(start, stop, step);
558
559                                a.fillDataset(n, iter);
560                                result.add(n);
561                                oind = ind;
562                        }
563                }
564
565                if (imax > oind) {
566                        nshape[axis] = imax - oind;
567                        start[axis] = oind;
568                        stop[axis] = imax;
569                        Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
570                        IndexIterator iter = a.getSliceIterator(start, stop, step);
571
572                        a.fillDataset(n, iter);
573                        result.add(n);
574                }
575
576                return result;
577        }
578
579        /**
580         * Constructs a dataset which has its elements along an axis replicated from
581         * the original dataset by the number of times given in the repeats array.
582         *
583         * By default, axis=-1 implies using a flattened version of the input dataset
584         *
585         * @param a
586         * @param repeats
587         * @param axis
588         * @return dataset
589         */
590        public static <T extends Dataset> T repeat(T a, int[] repeats, int axis) {
591                Serializable buf = a.getBuffer();
592                int[] shape = a.getShape();
593                int rank = shape.length;
594                final int is = a.getElementsPerItem();
595
596                if (axis >= rank) {
597                        utilsLogger.warn("Axis value is out of bounds");
598                        throw new IllegalArgumentException("Axis value is out of bounds");
599                }
600
601                int alen;
602                if (axis < 0) {
603                        alen = a.getSize();
604                        axis = 0;
605                        rank = 1;
606                        shape[0] = alen;
607                } else {
608                        alen = shape[axis];
609                }
610                int rlen = repeats.length;
611                if (rlen != 1 && rlen != alen) {
612                        utilsLogger.warn("Repeats array should have length of 1 or match chosen axis");
613                        throw new IllegalArgumentException("Repeats array should have length of 1 or match chosen axis");
614                }
615
616                for (int i = 0; i < rlen; i++) {
617                        if (repeats[i] < 0) {
618                                utilsLogger.warn("Negative repeat value is not allowed");
619                                throw new IllegalArgumentException("Negative repeat value is not allowed");
620                        }
621                }
622
623                int[] newShape = new int[rank];
624                for (int i = 0; i < rank; i ++)
625                        newShape[i] = shape[i];
626
627                // do single repeat separately
628                if (repeats.length == 1) {
629                        newShape[axis] *= repeats[0];
630                } else {
631                        int nlen = 0;
632                        for (int i = 0; i < alen; i++) {
633                                nlen += repeats[i];
634                        }
635                        newShape[axis] = nlen;
636                }
637
638                Dataset rdata = DatasetFactory.zeros(is, a.getClass(), newShape);
639                Serializable nbuf = rdata.getBuffer();
640
641                int csize = is; // chunk size
642                for (int i = axis+1; i < rank; i++) {
643                        csize *= newShape[i];
644                }
645                int nout = 1;
646                for (int i = 0; i < axis; i++) {
647                        nout *= newShape[i];
648                }
649
650                int oi = 0;
651                int ni = 0;
652                if (rlen == 1) { // do single repeat separately
653                        for (int i = 0; i < nout; i++) {
654                                for (int j = 0; j < shape[axis]; j++) {
655                                        for (int k = 0; k < repeats[0]; k++) {
656                                                System.arraycopy(buf, oi, nbuf, ni, csize);
657                                                ni += csize;
658                                        }
659                                        oi += csize;
660                                }
661                        }
662                } else {
663                        for (int i = 0; i < nout; i++) {
664                                for (int j = 0; j < shape[axis]; j++) {
665                                        for (int k = 0; k < repeats[j]; k++) {
666                                                System.arraycopy(buf, oi, nbuf, ni, csize);
667                                                ni += csize;
668                                        }
669                                        oi += csize;
670                                }
671                        }
672                }
673
674                return (T) rdata;
675        }
676
677        /**
678         * Resize a dataset
679         * @param a
680         * @param shape
681         * @return new dataset with new shape and items that are truncated or repeated, as necessary
682         */
683        public static <T extends Dataset> T resize(final T a, final int... shape) {
684                int size = a.getSize();
685                Dataset rdata = DatasetFactory.zeros(a.getElementsPerItem(), a.getClass(), shape);
686                IndexIterator it = rdata.getIterator();
687                while (it.hasNext()) {
688                        rdata.setObjectAbs(it.index, a.getObjectAbs(it.index % size));
689                }
690
691                return (T) rdata;
692        }
693
694        /**
695         * Copy and cast a dataset
696         *
697         * @param d
698         *            The dataset to be copied
699         * @param dtype dataset type
700         * @return copied dataset of given type
701         * @deprecated Please use the class-based methods in DatasetUtils,
702         *             such as {@link #copy(Class, IDataset)}
703         */
704        @Deprecated
705        public static Dataset copy(final IDataset d, final int dtype) {
706                return copy(DTypeUtils.getInterface(dtype), d);
707        }
708
709        /**
710         * Cast a dataset
711         *
712         * @param clazz dataset class
713         * @param d
714         *            The dataset to be cast.
715         * @return dataset of given class (or same dataset if already of the right class)
716         */
717        public static <T extends Dataset> T copy(Class<T> clazz, final IDataset d) {
718                Dataset a = convertToDataset(d);
719                Dataset c = null;
720                try {
721                        // copy across the data
722                        if (BooleanDataset.class.equals(clazz)) {
723                                c = new BooleanDataset(a);
724                        } else if (ByteDataset.class.equals(clazz)) {
725                                c = new ByteDataset(a);
726                        } else if (ShortDataset.class.equals(clazz)) {
727                                c = new ShortDataset(a);
728                        } else if (IntegerDataset.class.equals(clazz)) {
729                                c = new IntegerDataset(a);
730                        } else if (LongDataset.class.equals(clazz)) {
731                                c = new LongDataset(a);
732                        } else if (CompoundByteDataset.class.equals(clazz)) {
733                                if (a instanceof CompoundByteDataset) {
734                                        c = new CompoundByteDataset((CompoundDataset) a);
735                                } else {
736                                        c = new CompoundByteDataset(a);
737                                }
738                        } else if (CompoundShortDataset.class.equals(clazz)) {
739                                if (a instanceof CompoundDataset) {
740                                        c = new CompoundShortDataset((CompoundDataset) a);
741                                } else {
742                                        c = new CompoundShortDataset(a);
743                                }
744                        } else if (CompoundIntegerDataset.class.equals(clazz)) {
745                                if (a instanceof CompoundDataset) {
746                                        c = new CompoundIntegerDataset((CompoundDataset) a);
747                                } else {
748                                        c = new CompoundIntegerDataset(a);
749                                }
750                        } else if (CompoundLongDataset.class.equals(clazz)) {
751                                if (a instanceof CompoundDataset) {
752                                        c = new CompoundLongDataset((CompoundDataset) a);
753                                } else {
754                                        c = new CompoundLongDataset(a);
755                                }
756                        } else if (FloatDataset.class.equals(clazz)) {
757                                c = new FloatDataset(a);
758                        } else if (DoubleDataset.class.equals(clazz)) {
759                                c = new DoubleDataset(a);
760                        } else if (CompoundFloatDataset.class.equals(clazz)) {
761                                if (a instanceof CompoundDataset) {
762                                        c = new CompoundFloatDataset((CompoundDataset) a);
763                                } else {
764                                        c = new CompoundFloatDataset(a);
765                                }
766                        } else if (CompoundDoubleDataset.class.equals(clazz)) {
767                                if (a instanceof CompoundDataset) {
768                                        c = new CompoundDoubleDataset((CompoundDataset) a);
769                                } else {
770                                        c = new CompoundDoubleDataset(a);
771                                }
772                        } else if (ComplexFloatDataset.class.equals(clazz)) {
773                                c = new ComplexFloatDataset(a);
774                        } else if (ComplexDoubleDataset.class.equals(clazz)) {
775                                c = new ComplexDoubleDataset(a);
776                        } else if (RGBDataset.class.equals(clazz)) {
777                                if (a instanceof CompoundDataset) {
778                                        c = RGBDataset.createFromCompoundDataset((CompoundDataset) a);
779                                } else {
780                                        c = new RGBDataset(a);
781                                }
782                        } else if (StringDataset.class.equals(clazz)) {
783                                c = new StringDataset(a);
784                        } else if (DateDataset.class.equals(clazz)) {
785                                c = DateDatasetImpl.createFromObject(a);
786                        } else if (ObjectDataset.class.equals(clazz)) {
787                                c = new ObjectDataset(a);
788                        } else {
789                                utilsLogger.error("Dataset of unknown type!");
790                        }
791                } catch (OutOfMemoryError e) {
792                        utilsLogger.error("Not enough memory available to create dataset");
793                        throw new OutOfMemoryError("Not enough memory available to create dataset");
794                }
795
796                return (T) c;
797        }
798
799        /**
800         * Cast a dataset
801         *
802         * @param d
803         *            The dataset to be cast.
804         * @param dtype dataset type
805         * @return dataset of given type (or same dataset if already of the right type)
806         * @deprecated Please use the class-based methods in DatasetUtils,
807         *             such as {@link #cast(Class, IDataset)}
808         */
809        @Deprecated
810        public static Dataset cast(final IDataset d, final int dtype) {
811                return cast(DTypeUtils.getInterface(dtype), d);
812        }
813
814        /**
815         * Cast a dataset
816         * @param clazz dataset class
817         * @param d
818         *            The dataset to be cast.
819         * @return dataset of given class
820         */
821        public static <T extends Dataset> T cast(Class<T> clazz, final IDataset d) {
822                Dataset a = convertToDataset(d);
823
824                if ((a.getClass().equals(clazz))) {
825                        return (T) a;
826                }
827                return copy(clazz, d);
828        }
829
830        /**
831         * Cast a dataset
832         *
833         * @param d
834         *            The dataset to be cast.
835         * @param repeat repeat elements over item
836         * @param dtype dataset type
837         * @param isize item size
838         * @return dataset of given type
839         * @deprecated Please use the class-based methods in DatasetUtils,
840         *             such as {@link #cast(Class, IDataset)}
841         */
842        @Deprecated
843        public static Dataset cast(final IDataset d, final boolean repeat, final int dtype, final int isize) {
844                return cast(isize, DTypeUtils.getInterface(dtype), d, repeat);
845        }
846
847        /**
848         * Cast a dataset
849         *
850         * @param clazz dataset class
851         * @param d
852         *            The dataset to be cast.
853         * @param repeat repeat elements over item
854         * @param dtype dataset type
855         * @param isize item size
856         * @since 2.3
857         */
858        public static <T extends Dataset> T cast(final int isize, Class<T> clazz, final IDataset d, final boolean repeat) {
859                Dataset a = convertToDataset(d);
860
861                if (a.getClass().equals(clazz) && a.getElementsPerItem() == isize) {
862                        return (T) a;
863                }
864                if (isize <= 0) {
865                        utilsLogger.error("Item size is invalid (>0)");
866                        throw new IllegalArgumentException("Item size is invalid (>0)");
867                }
868                if (isize > 1 && !InterfaceUtils.isComplex(clazz) && InterfaceUtils.isElemental(clazz)) {
869                        utilsLogger.error("Item size is inconsistent with dataset type");
870                        throw new IllegalArgumentException("Item size is inconsistent with dataset type");
871                }
872
873                Dataset c = null;
874
875                try {
876                        // copy across the data
877                        if (BooleanDataset.class.equals(clazz)) {
878                                c = new BooleanDataset(a);
879                        } else if (ByteDataset.class.equals(clazz)) {
880                                c = new ByteDataset(a);
881                        } else if (ShortDataset.class.equals(clazz)) {
882                                c = new ShortDataset(a);
883                        } else if (IntegerDataset.class.equals(clazz)) {
884                                c = new IntegerDataset(a);
885                        } else if (LongDataset.class.equals(clazz)) {
886                                c = new LongDataset(a);
887                        } else if (CompoundByteDataset.class.equals(clazz)) {
888                                c = new CompoundByteDataset(isize, repeat, a);
889                        } else if (CompoundShortDataset.class.equals(clazz)) {
890                                c = new CompoundShortDataset(isize, repeat, a);
891                        } else if (CompoundIntegerDataset.class.equals(clazz)) {
892                                c = new CompoundIntegerDataset(isize, repeat, a);
893                        } else if (CompoundLongDataset.class.equals(clazz)) {
894                                c = new CompoundLongDataset(isize, repeat, a);
895                        } else if (FloatDataset.class.equals(clazz)) {
896                                c = new FloatDataset(a);
897                        } else if (DoubleDataset.class.equals(clazz)) {
898                                c = new DoubleDataset(a);
899                        } else if (CompoundFloatDataset.class.equals(clazz)) {
900                                c = new CompoundFloatDataset(isize, repeat, a);
901                        } else if (CompoundDoubleDataset.class.equals(clazz)) {
902                                c = new CompoundDoubleDataset(isize, repeat, a);
903                        } else if (ComplexFloatDataset.class.equals(clazz)) {
904                                c = new ComplexFloatDataset(a);
905                        } else if (ComplexDoubleDataset.class.equals(clazz)) {
906                                c = new ComplexDoubleDataset(a);
907                        } else if (RGBDataset.class.equals(clazz)) {
908                                if (a instanceof CompoundDataset) {
909                                        c = RGBDataset.createFromCompoundDataset((CompoundDataset) a);
910                                } else {
911                                        c = new RGBDataset(a);
912                                }
913                        } else if (StringDataset.class.equals(clazz)) {
914                                c = new StringDataset(a);
915                        } else if (DateDataset.class.equals(clazz)) {
916                                c = DateDatasetImpl.createFromObject(a);
917                        } else if (ObjectDataset.class.equals(clazz)) {
918                                c = new ObjectDataset(a);
919                        } else {
920                                utilsLogger.error("Dataset of unknown type!");
921                        }
922                } catch (OutOfMemoryError e) {
923                        utilsLogger.error("Not enough memory available to create dataset");
924                        throw new OutOfMemoryError("Not enough memory available to create dataset");
925                }
926
927                return (T) c;
928        }
929
930        /**
931         * Cast array of datasets to a compound dataset
932         *
933         * @param clazz dataset class
934         * @param a
935         *            The datasets to be cast.
936         * @return dataset of given class
937         * @since 2.3
938         */
939        public static  <T extends CompoundDataset> T compoundCast(Class<T> clazz, final Dataset... a) {
940                return (T) cast((Class<? extends Dataset>) clazz, a);
941        }
942
943
944        /**
945         * Cast array of datasets to a compound dataset
946         *
947         * @param clazz dataset class
948         * @param a
949         *            The datasets to be cast.
950         * @return compound dataset of given class
951         * @since 2.3
952         */
953        public static  CompoundDataset cast(Class<? extends Dataset> clazz, final Dataset... a) {
954                CompoundDataset c = null;
955                if (ByteDataset.class.equals(clazz) || CompoundByteDataset.class.equals(clazz)) {
956                        c = new CompoundByteDataset(a);
957                } else if (ShortDataset.class.equals(clazz) || CompoundShortDataset.class.equals(clazz)) {
958                        c = new CompoundShortDataset(a);
959                } else if (IntegerDataset.class.equals(clazz) || CompoundIntegerDataset.class.equals(clazz)) {
960                        c = new CompoundIntegerDataset(a);
961                } else if (LongDataset.class.equals(clazz) || CompoundLongDataset.class.equals(clazz)) {
962                        c = new CompoundLongDataset(a);
963                } else if (FloatDataset.class.equals(clazz) || CompoundFloatDataset.class.equals(clazz)) {
964                        c = new CompoundFloatDataset(a);
965                } else if (DoubleDataset.class.equals(clazz) || CompoundDoubleDataset.class.equals(clazz)) {
966                        c = new CompoundDoubleDataset(a);
967                } else if (ComplexFloatDataset.class.equals(clazz)) {
968                        if (a.length != 2) {
969                                throw new IllegalArgumentException("Need two datasets for complex dataset type");
970                        }
971                        c = new ComplexFloatDataset(a[0], a[1]);
972                } else if (ComplexDoubleDataset.class.equals(clazz)) {
973                        if (a.length != 2) {
974                                throw new IllegalArgumentException("Need two datasets for complex dataset type");
975                        }
976                        c = new ComplexDoubleDataset(a[0], a[1]);
977                } else {
978                        utilsLogger.error("Dataset of unsupported type!");
979                }
980
981                return c;
982        }
983
984        /**
985         * Cast array of datasets to a compound dataset
986         *
987         * @param a
988         *            The datasets to be cast.
989         * @return compound dataset of given type
990         * @deprecated Please use the class-based methods in DatasetUtils,
991         *             such as {@link #cast(Class, Dataset...)}
992         */
993        @Deprecated
994        public static CompoundDataset cast(final Dataset[] a, final int dtype) {
995                return cast(DTypeUtils.getInterface(dtype), a);
996        }
997
998        /**
999         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
1000         * of its contents
1001         * @param a
1002         * @return unsigned dataset or original if it is not an integer dataset
1003         */
1004        public static Dataset makeUnsigned(IDataset a) {
1005                return makeUnsigned(a, false);
1006        }
1007
1008        /**
1009         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
1010         * of its contents
1011         * @param a
1012         * @param check if true, then check for negative values
1013         * @return unsigned dataset or original if it is not an integer dataset or it has been check for negative numbers
1014         * @since 2.1
1015         */
1016        public static Dataset makeUnsigned(IDataset a, boolean check) {
1017                Dataset d = convertToDataset(a);
1018
1019                if (d.hasFloatingPointElements()) {
1020                        return d;
1021                }
1022                if (check && d.min(true).longValue() >= 0) {
1023                        return d;
1024                }
1025
1026                int dtype = d.getDType();
1027                switch (dtype) {
1028                case Dataset.INT32:
1029                        d = new LongDataset(d);
1030                        unwrapUnsigned(d, 32);
1031                        break;
1032                case Dataset.INT16:
1033                        d = new IntegerDataset(d);
1034                        unwrapUnsigned(d, 16);
1035                        break;
1036                case Dataset.INT8:
1037                        d = new ShortDataset(d);
1038                        unwrapUnsigned(d, 8);
1039                        break;
1040                case Dataset.ARRAYINT32:
1041                        d = new CompoundLongDataset(d);
1042                        unwrapUnsigned(d, 32);
1043                        break;
1044                case Dataset.ARRAYINT16:
1045                        d = new CompoundIntegerDataset(d);
1046                        unwrapUnsigned(d, 16);
1047                        break;
1048                case Dataset.ARRAYINT8:
1049                        d = new CompoundShortDataset(d);
1050                        unwrapUnsigned(d, 8);
1051                        break;
1052                }
1053                return d;
1054        }
1055
1056        /**
1057         * Unwrap dataset elements so that all elements are unsigned
1058         * @param a dataset
1059         * @param bitWidth width of original primitive in bits
1060         */
1061        public static void unwrapUnsigned(Dataset a, final int bitWidth) {
1062                final int dtype = a.getDType();
1063                final double dv = 1L << bitWidth;
1064                final int isize = a.getElementsPerItem();
1065                IndexIterator it = a.getIterator();
1066
1067                switch (dtype) {
1068                case Dataset.BOOL:
1069                        break;
1070                case Dataset.INT8:
1071                        break;
1072                case Dataset.INT16:
1073                        ShortDataset sds = (ShortDataset) a;
1074                        final short soffset = (short) dv;
1075                        while (it.hasNext()) {
1076                                final short x = sds.getAbs(it.index);
1077                                if (x < 0)
1078                                        sds.setAbs(it.index, (short) (x + soffset));
1079                        }
1080                        break;
1081                case Dataset.INT32:
1082                        IntegerDataset ids = (IntegerDataset) a;
1083                        final int ioffset = (int) dv;
1084                        while (it.hasNext()) {
1085                                final int x = ids.getAbs(it.index);
1086                                if (x < 0)
1087                                        ids.setAbs(it.index, x + ioffset);
1088                        }
1089                        break;
1090                case Dataset.INT64:
1091                        LongDataset lds = (LongDataset) a;
1092                        final long loffset = (long) dv;
1093                        while (it.hasNext()) {
1094                                final long x = lds.getAbs(it.index);
1095                                if (x < 0)
1096                                        lds.setAbs(it.index, x + loffset);
1097                        }
1098                        break;
1099                case Dataset.FLOAT32:
1100                        FloatDataset fds = (FloatDataset) a;
1101                        final float foffset = (float) dv;
1102                        while (it.hasNext()) {
1103                                final float x = fds.getAbs(it.index);
1104                                if (x < 0)
1105                                        fds.setAbs(it.index, x + foffset);
1106                        }
1107                        break;
1108                case Dataset.FLOAT64:
1109                        DoubleDataset dds = (DoubleDataset) a;
1110                        final double doffset = dv;
1111                        while (it.hasNext()) {
1112                                final double x = dds.getAbs(it.index);
1113                                if (x < 0)
1114                                        dds.setAbs(it.index, x + doffset);
1115                        }
1116                        break;
1117                case Dataset.ARRAYINT8:
1118                        break;
1119                case Dataset.ARRAYINT16:
1120                        CompoundShortDataset csds = (CompoundShortDataset) a;
1121                        final short csoffset = (short) dv;
1122                        final short[] csa = new short[isize];
1123                        while (it.hasNext()) {
1124                                csds.getAbs(it.index, csa);
1125                                boolean dirty = false;
1126                                for (int i = 0; i < isize; i++) {
1127                                        short x = csa[i];
1128                                        if (x < 0) {
1129                                                csa[i] = (short) (x + csoffset);
1130                                                dirty = true;
1131                                        }
1132                                }
1133                                if (dirty)
1134                                        csds.setAbs(it.index, csa);
1135                        }
1136                        break;
1137                case Dataset.ARRAYINT32:
1138                        CompoundIntegerDataset cids = (CompoundIntegerDataset) a;
1139                        final int cioffset = (int) dv;
1140                        final int[] cia = new int[isize];
1141                        while (it.hasNext()) {
1142                                cids.getAbs(it.index, cia);
1143                                boolean dirty = false;
1144                                for (int i = 0; i < isize; i++) {
1145                                        int x = cia[i];
1146                                        if (x < 0) {
1147                                                cia[i] = x + cioffset;
1148                                                dirty = true;
1149                                        }
1150                                }
1151                                if (dirty)
1152                                        cids.setAbs(it.index, cia);
1153                        }
1154                        break;
1155                case Dataset.ARRAYINT64:
1156                        CompoundLongDataset clds = (CompoundLongDataset) a;
1157                        final long cloffset = (long) dv;
1158                        final long[] cla = new long[isize];
1159                        while (it.hasNext()) {
1160                                clds.getAbs(it.index, cla);
1161                                boolean dirty = false;
1162                                for (int i = 0; i < isize; i++) {
1163                                        long x = cla[i];
1164                                        if (x < 0) {
1165                                                cla[i] = x + cloffset;
1166                                                dirty = true;
1167                                        }
1168                                }
1169                                if (dirty)
1170                                        clds.setAbs(it.index, cla);
1171                        }
1172                        break;
1173                default:
1174                        utilsLogger.error("Dataset of unsupported type for this method");
1175                        break;
1176                }
1177        }
1178
1179        /**
1180         * @param clazz dataset class
1181         * @param rows
1182         * @param cols
1183         * @param offset
1184         * @param dtype
1185         * @return a new 2d dataset of given shape and class, filled with ones on the (offset) diagonal
1186         * @since 2.3
1187         */
1188        public static <T extends Dataset> T eye(final Class<T> clazz, final int rows, final int cols, final int offset) {
1189                int[] shape = new int[] {rows, cols};
1190                T a = DatasetFactory.zeros(clazz, shape);
1191
1192                int[] pos = new int[] {0, offset};
1193                while (pos[1] < 0) {
1194                        pos[0]++;
1195                        pos[1]++;
1196                }
1197                while (pos[0] < rows && pos[1] < cols) {
1198                        a.set(1, pos);
1199                        pos[0]++;
1200                        pos[1]++;
1201                }
1202
1203                return a;
1204        }
1205
1206        /**
1207         * @param rows
1208         * @param cols
1209         * @param offset
1210         * @param dtype
1211         * @return a new 2d dataset of given shape and type, filled with ones on the (offset) diagonal
1212         * @deprecated Please use the class-based methods in DatasetUtils,
1213         *             such as {@link #eye(Class, int, int, int)}
1214         */
1215        @Deprecated
1216        public static Dataset eye(final int rows, final int cols, final int offset, final int dtype) {
1217                int[] shape = new int[] {rows, cols};
1218                Dataset a = DatasetFactory.zeros(shape, dtype);
1219
1220                int[] pos = new int[] {0, offset};
1221                while (pos[1] < 0) {
1222                        pos[0]++;
1223                        pos[1]++;
1224                }
1225                while (pos[0] < rows && pos[1] < cols) {
1226                        a.set(1, pos);
1227                        pos[0]++;
1228                        pos[1]++;
1229                }
1230
1231                return a;
1232        }
1233
1234        /**
1235         * Create a (off-)diagonal matrix from items in dataset
1236         * @param a
1237         * @param offset
1238         * @return diagonal matrix
1239         */
1240        public static <T extends Dataset> T diag(final T a, final int offset) {
1241                final int rank = a.getRank();
1242                final int is = a.getElementsPerItem();
1243
1244                if (rank == 0 || rank > 2) {
1245                        utilsLogger.error("Rank of dataset should be one or two");
1246                        throw new IllegalArgumentException("Rank of dataset should be one or two");
1247                }
1248
1249                Dataset result;
1250                final int[] shape = a.getShapeRef();
1251                if (rank == 1) {
1252                        int side = shape[0] + Math.abs(offset);
1253                        int[] pos = new int[] {side, side};
1254                        result = DatasetFactory.zeros(is, a.getClass(), pos);
1255                        if (offset >= 0) {
1256                                pos[0] = 0;
1257                                pos[1] = offset;
1258                        } else {
1259                                pos[0] = -offset;
1260                                pos[1] = 0;
1261                        }
1262                        int i = 0;
1263                        while (pos[0] < side && pos[1] < side) {
1264                                result.set(a.getObject(i++), pos);
1265                                pos[0]++;
1266                                pos[1]++;
1267                        }
1268                } else {
1269                        int side = offset >= 0 ? Math.min(shape[0], shape[1]-offset) : Math.min(shape[0]+offset, shape[1]);
1270                        if (side < 0)
1271                                side = 0;
1272                        result = DatasetFactory.zeros(is, a.getClass(), side);
1273
1274                        if (side > 0) {
1275                                int[] pos = offset >= 0 ? new int[] { 0, offset } : new int[] { -offset, 0 };
1276                                int i = 0;
1277                                while (pos[0] < shape[0] && pos[1] < shape[1]) {
1278                                        result.set(a.getObject(pos), i++);
1279                                        pos[0]++;
1280                                        pos[1]++;
1281                                }
1282                        }
1283                }
1284
1285                return (T) result;
1286        }
1287
1288        /**
1289         * Slice (or fully load), if necessary, a lazy dataset, otherwise take a slice view and
1290         * convert to our dataset implementation. If a slice is necessary, this may cause resource
1291         * problems when used on large datasets and throw runtime exceptions
1292         * @param lazy can be null
1293         * @return Converted dataset or null
1294         * @throws DatasetException
1295         */
1296        public static Dataset sliceAndConvertLazyDataset(ILazyDataset lazy) throws DatasetException {
1297                if (lazy == null)
1298                        return null;
1299
1300                IDataset data = lazy instanceof IDataset ? (IDataset) lazy.getSliceView() : lazy.getSlice();
1301
1302                return convertToDataset(data);
1303        }
1304
1305        /**
1306         * Convert (if necessary) a dataset obeying the interface to our implementation
1307         * @param data can be null
1308         * @return Converted dataset or null
1309         */
1310        public static Dataset convertToDataset(IDataset data) {
1311                if (data == null)
1312                        return null;
1313
1314                if (data instanceof Dataset) {
1315                        return (Dataset) data;
1316                }
1317
1318                final int isize = data.getElementsPerItem();
1319                Class<? extends Dataset> clazz = InterfaceUtils.getInterfaceFromClass(isize, data.getElementClass());
1320                if (isize <= 0) {
1321                        throw new IllegalArgumentException("Datasets with " + isize + " elements per item not supported");
1322                }
1323
1324                final Dataset result = DatasetFactory.zeros(isize, clazz, data.getShape());
1325                result.setName(data.getName());
1326
1327                final IndexIterator it = result.getIterator(true);
1328                final int[] pos = it.getPos();
1329                switch (DTypeUtils.getDType(clazz)) {
1330                case Dataset.BOOL:
1331                        while (it.hasNext()) {
1332                                result.setObjectAbs(it.index, data.getBoolean(pos));
1333                        }
1334                        break;
1335                case Dataset.INT8:
1336                        while (it.hasNext()) {
1337                                result.setObjectAbs(it.index, data.getByte(pos));
1338                        }
1339                        break;
1340                case Dataset.INT16:
1341                        while (it.hasNext()) {
1342                                result.setObjectAbs(it.index, data.getShort(pos));
1343                        }
1344                        break;
1345                case Dataset.INT32:
1346                        while (it.hasNext()) {
1347                                result.setObjectAbs(it.index, data.getInt(pos));
1348                        }
1349                        break;
1350                case Dataset.INT64:
1351                        while (it.hasNext()) {
1352                                result.setObjectAbs(it.index, data.getLong(pos));
1353                        }
1354                        break;
1355                case Dataset.FLOAT32:
1356                        while (it.hasNext()) {
1357                                result.setObjectAbs(it.index, data.getFloat(pos));
1358                        }
1359                        break;
1360                case Dataset.FLOAT64:
1361                        while (it.hasNext()) {
1362                                result.setObjectAbs(it.index, data.getDouble(pos));
1363                        }
1364                        break;
1365                default:
1366                        while (it.hasNext()) {
1367                                result.setObjectAbs(it.index, data.getObject(pos));
1368                        }
1369                        break;
1370                }
1371
1372                result.setErrors(data.getErrors());
1373                return result;
1374        }
1375
1376        /**
1377         * Create a compound dataset from given datasets
1378         * @param datasets
1379         * @return compound dataset or null if none given
1380         */
1381        public static CompoundDataset createCompoundDataset(final Dataset... datasets) {
1382                if (datasets == null || datasets.length == 0)
1383                        return null;
1384
1385                return createCompoundDataset(datasets[0].getDType(), datasets);
1386        }
1387
1388        /**
1389         * Create a compound dataset from given datasets
1390         * @param clazz dataset class
1391         * @param datasets
1392         * @return compound dataset or null if none given
1393         */
1394        public static <T extends CompoundDataset> T createCompoundDataset(Class<T> clazz, final Dataset... datasets) {
1395                return (T) createCompoundDataset(DTypeUtils.getDType(clazz), datasets);
1396        }
1397
1398        /**
1399         * Create a compound dataset from given datasets
1400         * @param dtype
1401         * @param datasets
1402         * @return compound dataset or null if none given
1403         * @deprecated Please use the class-based methods in DatasetUtils,
1404         *             such as {@link #createCompoundDataset(Class, Dataset...)}
1405         */
1406        @Deprecated
1407        public static CompoundDataset createCompoundDataset(final int dtype, final Dataset... datasets) {
1408                if (datasets == null || datasets.length == 0)
1409                        return null;
1410
1411                switch (dtype) {
1412                case Dataset.INT8:
1413                case Dataset.ARRAYINT8:
1414                        return new CompoundByteDataset(datasets);
1415                case Dataset.INT16:
1416                case Dataset.ARRAYINT16:
1417                        return new CompoundShortDataset(datasets);
1418                case Dataset.INT32:
1419                case Dataset.ARRAYINT32:
1420                        return new CompoundIntegerDataset(datasets);
1421                case Dataset.INT64:
1422                case Dataset.ARRAYINT64:
1423                        return new CompoundLongDataset(datasets);
1424                case Dataset.FLOAT32:
1425                case Dataset.ARRAYFLOAT32:
1426                        return new CompoundFloatDataset(datasets);
1427                case Dataset.FLOAT64:
1428                case Dataset.ARRAYFLOAT64:
1429                        return new CompoundDoubleDataset(datasets);
1430                case Dataset.COMPLEX64:
1431                case Dataset.COMPLEX128:
1432                        if (datasets.length > 2) {
1433                                utilsLogger.error("At most two datasets are allowed");
1434                                throw new IllegalArgumentException("At most two datasets are allowed");
1435                        } else if (datasets.length == 2) {
1436                                return dtype == Dataset.COMPLEX64 ? new ComplexFloatDataset(datasets[0], datasets[1]) : new ComplexDoubleDataset(datasets[0], datasets[1]);
1437                        }
1438                        return dtype == Dataset.COMPLEX64 ? new ComplexFloatDataset(datasets[0]) : new ComplexDoubleDataset(datasets[0]);
1439                case Dataset.RGB:
1440                        if (datasets.length == 1) {
1441                                return new RGBDataset(datasets[0]);
1442                        } else if (datasets.length == 3) {
1443                                return new RGBDataset(datasets[0], datasets[1], datasets[2]);
1444                        } else {
1445                                utilsLogger.error("Only one or three datasets are allowed to create a RGB dataset");
1446                                throw new IllegalArgumentException("Only one or three datasets are allowed to create a RGB dataset");
1447                        }
1448                default:
1449                        utilsLogger.error("Dataset type not supported for this operation");
1450                        throw new UnsupportedOperationException("Dataset type not supported");
1451                }
1452        }
1453
1454        /**
1455         * Create a compound dataset from given dataset
1456         * @param dataset
1457         * @param itemSize
1458         * @return compound dataset
1459         */
1460        public static CompoundDataset createCompoundDataset(final Dataset dataset, final int itemSize) {
1461                int[] shape = dataset.getShapeRef();
1462                int[] nshape = shape;
1463                if (shape != null && itemSize > 1) {
1464                        int size = ShapeUtils.calcSize(shape);
1465                        if (size % itemSize != 0) {
1466                                throw new IllegalArgumentException("Input dataset has number of items that is not a multiple of itemSize");
1467                        }
1468                        int d = shape.length;
1469                        int l = 1;
1470                        while (--d >= 0) {
1471                                l *= shape[d];
1472                                if (l % itemSize == 0) {
1473                                        break;
1474                                }
1475                        }
1476                        assert d >= 0;
1477                        nshape = new int[d + 1];
1478                        for (int i = 0; i < d; i++) {
1479                                nshape[i] = shape[i];
1480                        }
1481                        nshape[d] = l / itemSize;
1482                }
1483                switch (dataset.getDType()) {
1484                case Dataset.INT8:
1485                        return new CompoundByteDataset(itemSize, (byte[]) dataset.getBuffer(), nshape);
1486                case Dataset.INT16:
1487                        return new CompoundShortDataset(itemSize, (short[]) dataset.getBuffer(), nshape);
1488                case Dataset.INT32:
1489                        return new CompoundIntegerDataset(itemSize, (int[]) dataset.getBuffer(), nshape);
1490                case Dataset.INT64:
1491                        return new CompoundLongDataset(itemSize, (long[]) dataset.getBuffer(), nshape);
1492                case Dataset.FLOAT32:
1493                        return new CompoundFloatDataset(itemSize, (float[]) dataset.getBuffer(), nshape);
1494                case Dataset.FLOAT64:
1495                        return new CompoundDoubleDataset(itemSize, (double[]) dataset.getBuffer(), nshape);
1496                default:
1497                        utilsLogger.error("Dataset type not supported for this operation");
1498                        throw new UnsupportedOperationException("Dataset type not supported");
1499                }
1500        }
1501
1502
1503        /**
1504         * Create a compound dataset by using last axis as elements of an item
1505         * @param a
1506         * @param shareData if true, then share data
1507         * @return compound dataset
1508         */
1509        public static CompoundDataset createCompoundDatasetFromLastAxis(final Dataset a, final boolean shareData) {
1510                switch (a.getDType()) {
1511                case Dataset.INT8:
1512                        return CompoundByteDataset.createCompoundDatasetWithLastDimension(a, shareData);
1513                case Dataset.INT16:
1514                        return CompoundShortDataset.createCompoundDatasetWithLastDimension(a, shareData);
1515                case Dataset.INT32:
1516                        return CompoundIntegerDataset.createCompoundDatasetWithLastDimension(a, shareData);
1517                case Dataset.INT64:
1518                        return CompoundLongDataset.createCompoundDatasetWithLastDimension(a, shareData);
1519                case Dataset.FLOAT32:
1520                        return CompoundFloatDataset.createCompoundDatasetWithLastDimension(a, shareData);
1521                case Dataset.FLOAT64:
1522                        return CompoundDoubleDataset.createCompoundDatasetWithLastDimension(a, shareData);
1523                default:
1524                        utilsLogger.error("Dataset type not supported for this operation");
1525                        throw new UnsupportedOperationException("Dataset type not supported");
1526                }
1527        }
1528
1529        /**
1530         * Create a dataset from a compound dataset by using elements of an item as last axis
1531         * <p>
1532         * In the case where the number of elements is one, the last axis is squeezed out.
1533         * @param a
1534         * @param shareData if true, then share data
1535         * @return non-compound dataset
1536         */
1537        public static Dataset createDatasetFromCompoundDataset(final CompoundDataset a, final boolean shareData) {
1538                return a.asNonCompoundDataset(shareData);
1539        }
1540
1541        /**
1542         * Create a copy that has been coerced to an appropriate dataset type
1543         * depending on the input object's class
1544         *
1545         * @param a
1546         * @param obj
1547         * @return coerced copy of dataset
1548         */
1549        public static Dataset coerce(Dataset a, Object obj) {
1550                return cast(InterfaceUtils.getBestInterface(a.getClass(), InterfaceUtils.getInterface(obj)), a.clone());
1551        }
1552
1553        /**
1554         * Function that returns a normalised dataset which is bounded between 0 and 1
1555         * @param a dataset
1556         * @return normalised dataset
1557         */
1558        public static Dataset norm(Dataset a) {
1559                double amin = a.min().doubleValue();
1560                double aptp = a.max().doubleValue() - amin;
1561                Dataset temp = Maths.subtract(a, amin);
1562                temp.idivide(aptp);
1563                return temp;
1564        }
1565
1566        /**
1567         * Function that returns a normalised compound dataset which is bounded between 0 and 1. There
1568         * are (at least) two ways to normalise a compound dataset: per element - extrema for each element
1569         * in a compound item is used, i.e. many min/max pairs; over all elements - extrema for all elements
1570         * is used, i.e. one min/max pair.
1571         * @param a dataset
1572         * @param overAllElements if true, then normalise over all elements in each item
1573         * @return normalised dataset
1574         */
1575        public static CompoundDataset norm(CompoundDataset a, boolean overAllElements) {
1576                double[] amin = a.minItem();
1577                double[] amax = a.maxItem();
1578                final int is = a.getElementsPerItem();
1579                Dataset result;
1580
1581                if (overAllElements) {
1582                        Arrays.sort(amin);
1583                        Arrays.sort(amax);
1584                        double aptp = amax[0] - amin[0];
1585
1586                        result = Maths.subtract(a, amin[0]);
1587                        result.idivide(aptp);
1588                } else {
1589                        double[] aptp = new double[is];
1590                        for (int j = 0; j < is; j++) {
1591                                aptp[j] = amax[j] - amin[j];
1592                        }
1593
1594                        result = Maths.subtract(a, amin);
1595                        result.idivide(aptp);
1596                }
1597                return (CompoundDataset) result;
1598        }
1599
1600        /**
1601         * Function that returns a normalised dataset which is bounded between 0 and 1
1602         * and has been distributed on a log10 scale
1603         * @param a dataset
1604         * @return normalised dataset
1605         */
1606        public static Dataset lognorm(Dataset a) {
1607                double amin = a.min().doubleValue();
1608                double aptp = Math.log10(a.max().doubleValue() - amin + 1.);
1609                Dataset temp = Maths.subtract(a, amin - 1.);
1610                temp = Maths.log10(temp);
1611                temp = Maths.divide(temp, aptp);
1612                return temp;
1613        }
1614
1615        /**
1616         * Function that returns a normalised dataset which is bounded between 0 and 1
1617         * and has been distributed on a natural log scale
1618         * @param a dataset
1619         * @return normalised dataset
1620         */
1621        public static Dataset lnnorm(Dataset a) {
1622                double amin = a.min().doubleValue();
1623                double aptp = Math.log(a.max().doubleValue() - amin + 1.);
1624                Dataset temp = Maths.subtract(a, amin - 1.);
1625                temp = Maths.log(temp);
1626                temp = Maths.divide(temp, aptp);
1627                return temp;
1628        }
1629
1630        /**
1631         * Construct a list of datasets where each represents a coordinate varying over the hypergrid
1632         * formed by the input list of axes
1633         *
1634         * @param axes an array of 1D datasets representing axes
1635         * @return a list of coordinate datasets
1636         */
1637        public static List<Dataset> meshGrid(final Dataset... axes) {
1638                List<Dataset> result = new ArrayList<Dataset>();
1639                int rank = axes.length;
1640
1641                if (rank < 2) {
1642                        utilsLogger.error("Two or more axes datasets are required");
1643                        throw new IllegalArgumentException("Two or more axes datasets are required");
1644                }
1645
1646                int[] nshape = new int[rank];
1647
1648                for (int i = 0; i < rank; i++) {
1649                        Dataset axis = axes[i];
1650                        if (axis.getRank() != 1) {
1651                                utilsLogger.error("Given axis is not 1D");
1652                                throw new IllegalArgumentException("Given axis is not 1D");
1653                        }
1654                        nshape[i] = axis.getSize();
1655                }
1656
1657                for (int i = 0; i < rank; i++) {
1658                        Dataset axis = axes[i];
1659                        Dataset coord = DatasetFactory.zeros(axis.getClass(), nshape);
1660                        result.add(coord);
1661
1662                        final int alen = axis.getSize();
1663                        for (int j = 0; j < alen; j++) {
1664                                final Object obj = axis.getObjectAbs(j);
1665                                PositionIterator pi = coord.getPositionIterator(i);
1666                                final int[] pos = pi.getPos();
1667
1668                                pos[i] = j;
1669                                while (pi.hasNext()) {
1670                                        coord.set(obj, pos);
1671                                }
1672                        }
1673                }
1674
1675                return result;
1676        }
1677
1678        /**
1679         * Generate an index dataset for given dataset where sub-datasets contain index values
1680         *
1681         * @return an index dataset
1682         */
1683        public static IntegerDataset indices(int... shape) {
1684                // now create another dataset to plot against
1685                final int rank = shape.length;
1686                int[] nshape = new int[rank+1];
1687                nshape[0] = rank;
1688                for (int i = 0; i < rank; i++) {
1689                        nshape[i+1] = shape[i];
1690                }
1691
1692                IntegerDataset index = new IntegerDataset(nshape);
1693
1694                if (rank == 1) {
1695                        final int alen = shape[0];
1696                        int[] pos = new int[2];
1697                        for (int j = 0; j < alen; j++) {
1698                                pos[1] = j;
1699                                index.set(j, pos);
1700                        }
1701                } else {
1702                        for (int i = 1; i <= rank; i++) {
1703                                final int alen = nshape[i];
1704                                for (int j = 0; j < alen; j++) {
1705                                        PositionIterator pi = index.getPositionIterator(0, i);
1706                                        final int[] pos = pi.getPos();
1707
1708                                        pos[0] = i-1;
1709                                        pos[i] = j;
1710                                        while (pi.hasNext()) {
1711                                                index.set(j, pos);
1712                                        }
1713                                }
1714                        }
1715                }
1716                return index;
1717        }
1718
1719        /**
1720         * Get the centroid value of a dataset, this function works out the centroid in every direction
1721         *
1722         * @param a
1723         *            the dataset to be analysed
1724         * @param bases the optional array of base coordinates to use as weights.
1725         * This defaults to the mid-point of indices
1726         * @return a double array containing the centroid for each dimension
1727         */
1728        public static double[] centroid(Dataset a, Dataset... bases) {
1729                int rank = a.getRank();
1730                if (bases.length > 0 && bases.length != rank) {
1731                        throw new IllegalArgumentException("Number of bases must be zero or match rank of dataset");
1732                }
1733
1734                int[] shape = a.getShapeRef();
1735                if (bases.length == rank) {
1736                        for (int i = 0; i < rank; i++) {
1737                                Dataset b = bases[i];
1738                                if (b.getRank() != 1 && b.getSize() != shape[i]) {
1739                                        throw new IllegalArgumentException("A base does not have shape to match given dataset");
1740                                }
1741                        }
1742                }
1743
1744                double[] dc = new double[rank];
1745                if (rank == 0)
1746                        return dc;
1747
1748                final PositionIterator iter = new PositionIterator(shape);
1749                final int[] pos = iter.getPos();
1750
1751                double tsum = 0.0;
1752                while (iter.hasNext()) {
1753                        double val = a.getDouble(pos);
1754                        tsum += val;
1755                        for (int d = 0; d < rank; d++) {
1756                                Dataset b = bases.length == 0 ? null : bases[d];
1757                                if (b == null) {
1758                                        dc[d] += (pos[d] + 0.5) * val;
1759                                } else {
1760                                        dc[d] += b.getElementDoubleAbs(pos[d]) * val;
1761                                }
1762                        }
1763                }
1764
1765                for (int d = 0; d < rank; d++) {
1766                        dc[d] /= tsum;
1767                }
1768                return dc;
1769        }
1770
1771        /**
1772         * Find linearly-interpolated crossing points where the given dataset crosses the given value
1773         *
1774         * @param d
1775         * @param value
1776         * @return list of interpolated indices
1777         */
1778        public static List<Double> crossings(Dataset d, double value) {
1779                if (d.getRank() != 1) {
1780                        utilsLogger.error("Only 1d datasets supported");
1781                        throw new UnsupportedOperationException("Only 1d datasets supported");
1782                }
1783                List<Double> results = new ArrayList<Double>();
1784
1785                // run through all pairs of points on the line and see if value lies within
1786                IndexIterator it = d.getIterator();
1787                double y1, y2;
1788
1789                y2 = it.hasNext() ? d.getElementDoubleAbs(it.index) : 0;
1790                double x = 1;
1791                while (it.hasNext()) {
1792                        y1 = y2;
1793                        y2 = d.getElementDoubleAbs(it.index);
1794                        // check if value lies within pair [y1, y2]
1795                        if ((y1 <= value && value < y2) || (y1 > value && y2 <= value)) {
1796                                final double f = (value - y2)/(y2 - y1); // negative distance from right to left
1797                                results.add(x + f);
1798                        }
1799                        x++;
1800                }
1801                if (y2 == value) { // add end point of it intersects
1802                        results.add(x);
1803                }
1804
1805                return results;
1806        }
1807
1808        /**
1809         * Find x values of all the crossing points of the dataset with the given y value
1810         *
1811         * @param xAxis
1812         *            Dataset of the X axis that needs to be looked at
1813         * @param yAxis
1814         *            Dataset of the Y axis that needs to be looked at
1815         * @param yValue
1816         *            The y value the X values are required for
1817         * @return An list of doubles containing all the X coordinates of where the line crosses
1818         */
1819        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue) {
1820                if (xAxis.getSize() > yAxis.getSize()) {
1821                        throw new IllegalArgumentException(
1822                                        "Number of values of yAxis must as least be equal to the number of values of xAxis");
1823                }
1824
1825                List<Double> results = new ArrayList<Double>();
1826
1827                List<Double> indices = crossings(yAxis, yValue);
1828
1829                for (double xi : indices) {
1830                        results.add(Maths.interpolate(xAxis, xi));
1831                }
1832                return results;
1833        }
1834
1835        /**
1836         * Function that uses the crossings function but prunes the result, so that multiple crossings within a
1837         * certain proportion of the overall range of the x values
1838         *
1839         * @param xAxis
1840         *            Dataset of the X axis
1841         * @param yAxis
1842         *            Dataset of the Y axis
1843         * @param yValue
1844         *            The y value the x values are required for
1845         * @param xRangeProportion
1846         *            The proportion of the overall x spread used to prune result
1847         * @return A list containing all the unique crossing points
1848         */
1849        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue, double xRangeProportion) {
1850                // get the values found
1851                List<Double> vals = crossings(xAxis, yAxis, yValue);
1852
1853                // use the proportion to calculate the error spacing
1854                double error = xRangeProportion * xAxis.peakToPeak().doubleValue();
1855
1856                int i = 0;
1857                // now go through and check for groups of three crossings which are all
1858                // within the boundaries
1859                while (i <= vals.size() - 3) {
1860                        double v1 = Math.abs(vals.get(i) - vals.get(i + 2));
1861                        if (v1 < error) {
1862                                // these 3 points should be treated as one
1863                                // make the first point equal to the average of them all
1864                                vals.set(i + 2, ((vals.get(i) + vals.get(i + 1) + vals.get(i + 2)) / 3.0));
1865                                // remove the other offending points
1866                                vals.remove(i);
1867                                vals.remove(i);
1868                        } else {
1869                                i++;
1870                        }
1871                }
1872
1873                // once the thinning process has been completed, return the pruned list
1874                return vals;
1875        }
1876
1877        // recursive function
1878        private static void setRow(Object row, Dataset a, int... pos) {
1879                final int l = Array.getLength(row);
1880                final int rank = pos.length;
1881                final int[] npos = Arrays.copyOf(pos, rank+1);
1882                Object r;
1883                if (rank+1 < a.getRank()) {
1884                        for (int i = 0; i < l; i++) {
1885                                npos[rank] = i;
1886                                r = Array.get(row, i);
1887                                setRow(r, a, npos);
1888                        }
1889                } else {
1890                        for (int i = 0; i < l; i++) {
1891                                npos[rank] = i;
1892                                r = a.getObject(npos);
1893                                Array.set(row, i, r);
1894                        }
1895                }
1896        }
1897
1898        /**
1899         * Create Java array (of arrays) from dataset
1900         * @param a dataset
1901         * @return Java array (of arrays...)
1902         */
1903        public static Object createJavaArray(Dataset a) {
1904                if (a.getElementsPerItem() > 1) {
1905                        a = createDatasetFromCompoundDataset((CompoundDataset) a, true);
1906                }
1907                Object matrix;
1908
1909                int[] shape = a.getShapeRef();
1910                switch (a.getDType()) {
1911                case Dataset.BOOL:
1912                        matrix = Array.newInstance(boolean.class, shape);
1913                        break;
1914                case Dataset.INT8:
1915                        matrix = Array.newInstance(byte.class, shape);
1916                        break;
1917                case Dataset.INT16:
1918                        matrix = Array.newInstance(short.class, shape);
1919                        break;
1920                case Dataset.INT32:
1921                        matrix = Array.newInstance(int.class, shape);
1922                        break;
1923                case Dataset.INT64:
1924                        matrix = Array.newInstance(long.class, shape);
1925                        break;
1926                case Dataset.FLOAT32:
1927                        matrix = Array.newInstance(float.class, shape);
1928                        break;
1929                case Dataset.FLOAT64:
1930                        matrix = Array.newInstance(double.class, shape);
1931                        break;
1932                default:
1933                        utilsLogger.error("Dataset type not supported");
1934                        throw new IllegalArgumentException("Dataset type not supported");
1935                }
1936
1937                // populate matrix
1938                setRow(matrix, a);
1939                return matrix;
1940        }
1941
1942        /**
1943         * Removes NaNs and infinities from floating point datasets.
1944         * All other dataset types are ignored.
1945         *
1946         * @param a dataset
1947         * @param value replacement value
1948         */
1949        public static void removeNansAndInfinities(Dataset a, final Number value) {
1950                if (a instanceof DoubleDataset) {
1951                        final double dvalue = DTypeUtils.toReal(value);
1952                        final DoubleDataset set = (DoubleDataset) a;
1953                        final IndexIterator it = set.getIterator();
1954                        final double[] data = set.getData();
1955                        while (it.hasNext()) {
1956                                double x = data[it.index];
1957                                if (Double.isNaN(x) || Double.isInfinite(x))
1958                                        data[it.index] = dvalue;
1959                        }
1960                } else if (a instanceof FloatDataset) {
1961                        final float fvalue = (float) DTypeUtils.toReal(value);
1962                        final FloatDataset set = (FloatDataset) a;
1963                        final IndexIterator it = set.getIterator();
1964                        final float[] data = set.getData();
1965                        while (it.hasNext()) {
1966                                float x = data[it.index];
1967                                if (Float.isNaN(x) || Float.isInfinite(x))
1968                                        data[it.index] = fvalue;
1969                        }
1970                } else if (a instanceof CompoundDoubleDataset) {
1971                        final double dvalue = DTypeUtils.toReal(value);
1972                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
1973                        final int is = set.getElementsPerItem();
1974                        final IndexIterator it = set.getIterator();
1975                        final double[] data = set.getData();
1976                        while (it.hasNext()) {
1977                                for (int j = 0; j < is; j++) {
1978                                        double x = data[it.index + j];
1979                                        if (Double.isNaN(x) || Double.isInfinite(x))
1980                                                data[it.index + j] = dvalue;
1981                                }
1982                        }
1983                } else if (a instanceof CompoundFloatDataset) {
1984                        final float fvalue = (float) DTypeUtils.toReal(value);
1985                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
1986                        final int is = set.getElementsPerItem();
1987                        final IndexIterator it = set.getIterator();
1988                        final float[] data = set.getData();
1989                        while (it.hasNext()) {
1990                                for (int j = 0; j < is; j++) {
1991                                        float x = data[it.index + j];
1992                                        if (Float.isNaN(x) || Float.isInfinite(x))
1993                                                data[it.index + j] = fvalue;
1994                                }
1995                        }
1996                }
1997        }
1998
1999        /**
2000         * Make floating point datasets contain only finite values. Infinities and NaNs are replaced
2001         * by +/- MAX_VALUE and 0, respectively.
2002         * All other dataset types are ignored.
2003         *
2004         * @param a dataset
2005         */
2006        public static void makeFinite(Dataset a) {
2007                if (a instanceof DoubleDataset) {
2008                        final DoubleDataset set = (DoubleDataset) a;
2009                        final IndexIterator it = set.getIterator();
2010                        final double[] data = set.getData();
2011                        while (it.hasNext()) {
2012                                final double x = data[it.index];
2013                                if (Double.isNaN(x))
2014                                        data[it.index] = 0;
2015                                else if (Double.isInfinite(x))
2016                                        data[it.index] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
2017                        }
2018                } else if (a instanceof FloatDataset) {
2019                        final FloatDataset set = (FloatDataset) a;
2020                        final IndexIterator it = set.getIterator();
2021                        final float[] data = set.getData();
2022                        while (it.hasNext()) {
2023                                final float x = data[it.index];
2024                                if (Float.isNaN(x))
2025                                        data[it.index] = 0;
2026                                else if (Float.isInfinite(x))
2027                                        data[it.index] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
2028                        }
2029                } else if (a instanceof CompoundDoubleDataset) {
2030                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
2031                        final int is = set.getElementsPerItem();
2032                        final IndexIterator it = set.getIterator();
2033                        final double[] data = set.getData();
2034                        while (it.hasNext()) {
2035                                for (int j = 0; j < is; j++) {
2036                                        final double x = data[it.index + j];
2037                                        if (Double.isNaN(x))
2038                                                data[it.index + j] = 0;
2039                                        else if (Double.isInfinite(x))
2040                                                data[it.index + j] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
2041                                }
2042                        }
2043                } else if (a instanceof CompoundFloatDataset) {
2044                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
2045                        final int is = set.getElementsPerItem();
2046                        final IndexIterator it = set.getIterator();
2047                        final float[] data = set.getData();
2048                        while (it.hasNext()) {
2049                                for (int j = 0; j < is; j++) {
2050                                        final float x = data[it.index + j];
2051                                        if (Float.isNaN(x))
2052                                                data[it.index + j] = 0;
2053                                        else if (Float.isInfinite(x))
2054                                                data[it.index + j] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
2055                                }
2056                        }
2057                }
2058        }
2059
2060        /**
2061         * Find absolute index of first value in dataset that is equal to given number
2062         * @param a
2063         * @param n
2064         * @return absolute index (if greater than a.getSize() then no value found)
2065         */
2066        public static int findIndexEqualTo(final Dataset a, final double n) {
2067                IndexIterator iter = a.getIterator();
2068                while (iter.hasNext()) {
2069                        if (a.getElementDoubleAbs(iter.index) == n)
2070                                break;
2071                }
2072
2073                return iter.index;
2074        }
2075
2076        /**
2077         * Find absolute index of first value in dataset that is greater than given number
2078         * @param a
2079         * @param n
2080         * @return absolute index (if greater than a.getSize() then no value found)
2081         */
2082        public static int findIndexGreaterThan(final Dataset a, final double n) {
2083                IndexIterator iter = a.getIterator();
2084                while (iter.hasNext()) {
2085                        if (a.getElementDoubleAbs(iter.index) > n)
2086                                break;
2087                }
2088
2089                return iter.index;
2090        }
2091
2092        /**
2093         * Find absolute index of first value in dataset that is greater than or equal to given number
2094         * @param a
2095         * @param n
2096         * @return absolute index (if greater than a.getSize() then no value found)
2097         */
2098        public static int findIndexGreaterThanOrEqualTo(final Dataset a, final double n) {
2099                IndexIterator iter = a.getIterator();
2100                while (iter.hasNext()) {
2101                        if (a.getElementDoubleAbs(iter.index) >= n)
2102                                break;
2103                }
2104
2105                return iter.index;
2106        }
2107
2108        /**
2109         * Find absolute index of first value in dataset that is less than given number
2110         * @param a
2111         * @param n
2112         * @return absolute index (if greater than a.getSize() then no value found)
2113         */
2114        public static int findIndexLessThan(final Dataset a, final double n) {
2115                IndexIterator iter = a.getIterator();
2116                while (iter.hasNext()) {
2117                        if (a.getElementDoubleAbs(iter.index) < n)
2118                                break;
2119                }
2120
2121                return iter.index;
2122        }
2123
2124        /**
2125         * Find absolute index of first value in dataset that is less than or equal to given number
2126         * @param a
2127         * @param n
2128         * @return absolute index (if greater than a.getSize() then no value found)
2129         */
2130        public static int findIndexLessThanOrEqualTo(final Dataset a, final double n) {
2131                IndexIterator iter = a.getIterator();
2132                while (iter.hasNext()) {
2133                        if (a.getElementDoubleAbs(iter.index) <= n)
2134                                break;
2135                }
2136
2137                return iter.index;
2138        }
2139
2140        /**
2141         * Find first occurrences in one dataset of values given in another sorted dataset
2142         * @param a
2143         * @param values sorted 1D dataset of values to find
2144         * @return absolute indexes of those first occurrences (-1 is used to indicate value not found)
2145         */
2146        public static IntegerDataset findFirstOccurrences(final Dataset a, final Dataset values) {
2147                if (values.getRank() != 1) {
2148                        throw new IllegalArgumentException("Values dataset must be 1D");
2149                }
2150                IntegerDataset indexes = new IntegerDataset(values.getSize());
2151                indexes.fill(-1);
2152
2153                IndexIterator it = a.getIterator();
2154                final int n = values.getSize();
2155                if (values instanceof LongDataset) {
2156                        while (it.hasNext()) {
2157                                long x = a.getElementLongAbs(it.index);
2158
2159                                int l = 0; // binary search to find value in sorted dataset
2160                                long vl = values.getLong(l);
2161                                if (x <= vl) {
2162                                        if (x == vl && indexes.getAbs(l) < 0)
2163                                                indexes.setAbs(l, it.index);
2164                                        continue;
2165                                }
2166                                int h = n - 1;
2167                                long vh = values.getLong(h);
2168                                if (x >= vh) {
2169                                        if (x == vh && indexes.getAbs(h) < 0)
2170                                                indexes.setAbs(h, it.index);
2171                                        continue;
2172                                }
2173                                while (h - l > 1) {
2174                                        int m = (l + h) / 2;
2175                                        long vm = values.getLong(m);
2176                                        if (x < vm) {
2177                                                h = m;
2178                                        } else if (x > vm) {
2179                                                l = m;
2180                                        } else {
2181                                                if (indexes.getAbs(m) < 0)
2182                                                        indexes.setAbs(m, it.index);
2183                                                break;
2184                                        }
2185                                }
2186                        }
2187                } else {
2188                        while (it.hasNext()) {
2189                                double x = a.getElementDoubleAbs(it.index);
2190
2191                                int l = 0; // binary search to find value in sorted dataset
2192                                double vl = values.getDouble(l);
2193                                if (x <= vl) {
2194                                        if (x == vl && indexes.getAbs(l) < 0)
2195                                                indexes.setAbs(l, it.index);
2196                                        continue;
2197                                }
2198                                int h = n - 1;
2199                                double vh = values.getDouble(h);
2200                                if (x >= vh) {
2201                                        if (x == vh && indexes.getAbs(h) < 0)
2202                                                indexes.setAbs(h, it.index);
2203                                        continue;
2204                                }
2205                                while (h - l > 1) {
2206                                        int m = (l + h) / 2;
2207                                        double vm = values.getDouble(m);
2208                                        if (x < vm) {
2209                                                h = m;
2210                                        } else if (x > vm) {
2211                                                l = m;
2212                                        } else {
2213                                                if (indexes.getAbs(m) < 0)
2214                                                        indexes.setAbs(m, it.index);
2215                                                break;
2216                                        }
2217                                }
2218                        }
2219                }
2220                return indexes;
2221        }
2222
2223        /**
2224         * Find indexes in sorted dataset of values for each value in other dataset
2225         * @param a
2226         * @param values sorted 1D dataset of values to find
2227         * @return absolute indexes of values (-1 is used to indicate value not found)
2228         */
2229        public static IntegerDataset findIndexesForValues(final Dataset a, final Dataset values) {
2230                if (values.getRank() != 1) {
2231                        throw new IllegalArgumentException("Values dataset must be 1D");
2232                }
2233                IntegerDataset indexes = new IntegerDataset(a.getSize());
2234                indexes.fill(-1);
2235
2236                IndexIterator it = a.getIterator();
2237                int i = -1;
2238                final int n = values.getSize();
2239                if (values instanceof LongDataset) {
2240                        while (it.hasNext()) {
2241                                i++;
2242                                long x = a.getElementLongAbs(it.index);
2243
2244                                int l = 0; // binary search to find value in sorted dataset
2245                                long vl = values.getLong(l);
2246                                if (x <= vl) {
2247                                        if (x == vl)
2248                                                indexes.setAbs(i, l);
2249                                        continue;
2250                                }
2251                                int h = n - 1;
2252                                long vh = values.getLong(h);
2253                                if (x >= vh) {
2254                                        if (x == vh)
2255                                                indexes.setAbs(i, h);
2256                                        continue;
2257                                }
2258                                while (h - l > 1) {
2259                                        int m = (l + h) / 2;
2260                                        long vm = values.getLong(m);
2261                                        if (x < vm) {
2262                                                h = m;
2263                                        } else if (x > vm) {
2264                                                l = m;
2265                                        } else {
2266                                                indexes.setAbs(i, m);
2267                                                break;
2268                                        }
2269                                }
2270                        }
2271                } else {
2272                        while (it.hasNext()) {
2273                                i++;
2274                                double x = a.getElementDoubleAbs(it.index);
2275
2276                                int l = 0; // binary search to find value in sorted dataset
2277                                double vl = values.getDouble(l);
2278                                if (x <= vl) {
2279                                        if (x == vl)
2280                                                indexes.setAbs(i, l);
2281                                        continue;
2282                                }
2283                                int h = n - 1;
2284                                double vh = values.getDouble(h);
2285                                if (x >= vh) {
2286                                        if (x == vh)
2287                                                indexes.setAbs(i, h);
2288                                        continue;
2289                                }
2290                                while (h - l > 1) {
2291                                        int m = (l + h) / 2;
2292                                        double vm = values.getDouble(m);
2293                                        if (x < vm) {
2294                                                h = m;
2295                                        } else if (x > vm) {
2296                                                l = m;
2297                                        } else {
2298                                                indexes.setAbs(i, m);
2299                                                break;
2300                                        }
2301                                }
2302                        }
2303                }
2304
2305                return indexes;
2306        }
2307
2308        /**
2309         * Roll items over given axis by given amount
2310         * @param a
2311         * @param shift
2312         * @param axis if null, then roll flattened dataset
2313         * @return rolled dataset
2314         */
2315        public static <T extends Dataset> T roll(final T a, final int shift, Integer axis) {
2316                Dataset r = DatasetFactory.zeros(a);
2317                int is = a.getElementsPerItem();
2318                if (axis == null) {
2319                        IndexIterator it = a.getIterator();
2320                        int s = r.getSize();
2321                        int i = shift % s;
2322                        if (i < 0)
2323                                i += s;
2324                        while (it.hasNext()) {
2325                                r.setObjectAbs(i, a.getObjectAbs(it.index));
2326                                i += is;
2327                                if (i >= s) {
2328                                        i %= s;
2329                                }
2330                        }
2331                } else {
2332                        axis = a.checkAxis(axis);
2333                        PositionIterator pi = a.getPositionIterator(axis);
2334                        int s = a.getShapeRef()[axis];
2335                        Dataset u = DatasetFactory.zeros(is, a.getClass(), new int[] {s});
2336                        Dataset v = DatasetFactory.zeros(u);
2337                        int[] pos = pi.getPos();
2338                        boolean[] hit = pi.getOmit();
2339                        while (pi.hasNext()) {
2340                                a.copyItemsFromAxes(pos, hit, u);
2341                                int i = shift % s;
2342                                if (i < 0)
2343                                        i += s;
2344                                for (int j = 0; j < s; j++) {
2345                                        v.setObjectAbs(i, u.getObjectAbs(j*is));
2346                                        i += is;
2347                                        if (i >= s) {
2348                                                i %= s;
2349                                        }
2350                                }
2351                                r.setItemsOnAxes(pos, hit, v.getBuffer());
2352                        }
2353                }
2354                return (T) r;
2355        }
2356
2357        /**
2358         * Roll the specified axis backwards until it lies in given position
2359         * @param a
2360         * @param axis The rolled axis (index in shape array). Other axes are left unchanged in relative positions
2361         * @param start The position with it right of the destination of the rolled axis
2362         * @return dataset with rolled axis
2363         */
2364        public static <T extends Dataset> T rollAxis(final T a, int axis, int start) {
2365                int r = a.getRank();
2366                axis = a.checkAxis(axis);
2367                if (start < 0)
2368                        start += r;
2369                if (start < 0 || start > r) {
2370                        throw new IllegalArgumentException("Start is out of range: it should be >= 0 and <= " + r);
2371                }
2372                if (axis < start)
2373                        start--;
2374
2375                if (axis == start)
2376                        return a;
2377
2378                ArrayList<Integer> axes = new ArrayList<Integer>();
2379                for (int i = 0; i < r; i++) {
2380                        if (i != axis) {
2381                                axes.add(i);
2382                        }
2383                }
2384                axes.add(start, axis);
2385                int[] aa = new int[r];
2386                for (int i = 0; i < r; i++) {
2387                        aa[i] = axes.get(i);
2388                }
2389                return (T) a.getTransposedView(aa);
2390        }
2391
2392        private static SliceND createFlippedSlice(final Dataset a, int axis) {
2393                int[] shape = a.getShapeRef();
2394                SliceND slice = new SliceND(shape);
2395                slice.flip(axis);
2396                return slice;
2397        }
2398
2399        /**
2400         * Flip items in left/right direction, column-wise, or along second axis
2401         * @param a dataset must be at least 2D
2402         * @return view of flipped dataset
2403         */
2404        public static <T extends Dataset> T flipLeftRight(final T a) {
2405                if (a.getRank() < 2) {
2406                        throw new IllegalArgumentException("Dataset must be at least 2D");
2407                }
2408                return (T) a.getSliceView(createFlippedSlice(a, 1));
2409        }
2410
2411        /**
2412         * Flip items in up/down direction, row-wise, or along first axis
2413         * @param a dataset
2414         * @return view of flipped dataset
2415         */
2416        public static <T extends Dataset> T flipUpDown(final T a) {
2417                return (T) a.getSliceView(createFlippedSlice(a, 0));
2418        }
2419
2420        /**
2421         * Rotate items in first two dimension by 90 degrees anti-clockwise
2422         * @param a dataset must be at least 2D
2423         * @return view of flipped dataset
2424         */
2425        public static <T extends Dataset> T rotate90(final T a) {
2426                return rotate90(a, 1);
2427        }
2428
2429        /**
2430         * Rotate items in first two dimension by 90 degrees anti-clockwise
2431         * @param a dataset must be at least 2D
2432         * @param k number of 90-degree rotations
2433         * @return view of flipped dataset
2434         */
2435        public static <T extends Dataset> T rotate90(final T a, int k) {
2436                k = k % 4;
2437                while (k < 0) {
2438                        k += 4;
2439                }
2440                int r = a.getRank();
2441                if (r < 2) {
2442                        throw new IllegalArgumentException("Dataset must be at least 2D");
2443                }
2444                switch (k) {
2445                case 1: case 3:
2446                        int[] axes = new int[r];
2447                        axes[0] = 1;
2448                        axes[1] = 0;
2449                        for (int i = 2; i < r; i++) {
2450                                axes[i] = i;
2451                        }
2452                        Dataset t = a.getTransposedView(axes);
2453                        return (T) t.getSliceView(createFlippedSlice(t, k == 1 ? 0 : 1));
2454                case 2:
2455                        SliceND s = createFlippedSlice(a, 0);
2456                        s.flip(1);
2457                        return (T) a.getSliceView(s);
2458                default:
2459                case 0:
2460                        return a;
2461                }
2462        }
2463
2464        /**
2465         * Select content according where condition is true. All inputs are broadcasted to a maximum shape
2466         * @param condition boolean dataset
2467         * @param x
2468         * @param y
2469         * @return dataset where content is x or y depending on whether condition is true or otherwise
2470         */
2471        public static Dataset select(BooleanDataset condition, Object x, Object y) {
2472                Object[] all = new Object[] {condition, x, y};
2473                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
2474                condition = (BooleanDataset) dAll[0];
2475                Dataset dx = dAll[1];
2476                Dataset dy = dAll[2];
2477                Class<? extends Dataset> dc = InterfaceUtils.getBestInterface(dx.getClass(), dy.getClass());
2478                int ds = Math.max(dx.getElementsPerItem(), dy.getElementsPerItem());
2479
2480                Dataset r = DatasetFactory.zeros(ds, dc, condition.getShapeRef());
2481                IndexIterator iter = condition.getIterator(true);
2482                final int[] pos = iter.getPos();
2483                int i = 0;
2484                while (iter.hasNext()) {
2485                        r.setObjectAbs(i++, condition.getElementBooleanAbs(iter.index) ? dx.getObject(pos) : dy.getObject(pos));
2486                }
2487                return r;
2488        }
2489
2490        /**
2491         * Select content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
2492         * @param conditions array of boolean datasets
2493         * @param choices array of datasets or objects
2494         * @param def default value (can be a dataset)
2495         * @return dataset
2496         */
2497        public static Dataset select(BooleanDataset[] conditions, Object[] choices, Object def) {
2498                final int n = conditions.length;
2499                if (choices.length != n) {
2500                        throw new IllegalArgumentException("Choices list is not same length as conditions list");
2501                }
2502                Object[] all = new Object[2*n];
2503                System.arraycopy(conditions, 0, all, 0, n);
2504                System.arraycopy(choices, 0, all, n, n);
2505                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
2506                conditions = new BooleanDataset[n];
2507                Dataset[] dChoices = new Dataset[n];
2508                System.arraycopy(dAll, 0, conditions, 0, n);
2509                System.arraycopy(dAll, n, dChoices, 0, n);
2510                Class<? extends Dataset> dc = null;
2511                int ds = -1;
2512                for (int i = 0; i < n; i++) {
2513                        Dataset a = dChoices[i];
2514                        Class<? extends Dataset> c = a.getClass();
2515                        dc = InterfaceUtils.getBestInterface(dc, c);
2516                        int s = a.getElementsPerItem();
2517                        if (s > ds) {
2518                                ds = s;
2519                }
2520                }
2521                if (dc == null || ds < 1) {
2522                        throw new IllegalArgumentException("Dataset types of choices are invalid");
2523                }
2524
2525                Dataset r = DatasetFactory.zeros(ds, dc, conditions[0].getShapeRef());
2526                Dataset d = DatasetFactory.createFromObject(def).getBroadcastView(r.getShapeRef());
2527                PositionIterator iter = new PositionIterator(r.getShapeRef());
2528                final int[] pos = iter.getPos();
2529                int i = 0;
2530                while (iter.hasNext()) {
2531                        int j = 0;
2532                        for (; j < n; j++) {
2533                                if (conditions[j].get(pos)) {
2534                                        r.setObjectAbs(i++, dChoices[j].getObject(pos));
2535                                        break;
2536                                }
2537                        }
2538                        if (j == n) {
2539                                r.setObjectAbs(i++, d.getObject(pos));
2540                        }
2541                }
2542                return r;
2543        }
2544
2545        /**
2546         * Choose content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
2547         * @param index integer dataset (ideally, items should be in [0, n) range, if there are n choices)
2548         * @param choices array of datasets or objects
2549         * @param throwAIOOBE if true, throw array index out of bound exception
2550         * @param clip true to clip else wrap indices out of bounds; only used when throwAOOBE is false
2551         * @return dataset
2552         */
2553        public static Dataset choose(IntegerDataset index, Object[] choices, boolean throwAIOOBE, boolean clip) {
2554                final int n = choices.length;
2555                Object[] all = new Object[n + 1];
2556                System.arraycopy(choices, 0, all, 0, n);
2557                all[n] = index;
2558                Dataset[] dChoices = BroadcastUtils.convertAndBroadcast(all);
2559                Class<? extends Dataset> dc = null;
2560                int ds = -1;
2561                int mr = -1;
2562                for (int i = 0; i < n; i++) {
2563                        Dataset a = dChoices[i];
2564                        int r = a.getRank();
2565                        if (r > mr) {
2566                                mr = r;
2567                        }
2568                        dc = InterfaceUtils.getBestInterface(dc, a.getClass());
2569                        int s = a.getElementsPerItem();
2570                        if (s > ds) {
2571                                ds = s;
2572                }
2573                }
2574                if (dc == null || ds < 1) {
2575                        throw new IllegalArgumentException("Dataset types of choices are invalid");
2576                }
2577                index = (IntegerDataset) dChoices[n];
2578                dChoices[n] = null;
2579
2580                Dataset r = DatasetFactory.zeros(ds, dc, index.getShapeRef());
2581                IndexIterator iter = index.getIterator(true);
2582                final int[] pos = iter.getPos();
2583                int i = 0;
2584                while (iter.hasNext()) {
2585                        int j = index.getAbs(iter.index);
2586                        if (j < 0) {
2587                                if (throwAIOOBE)
2588                                        throw new ArrayIndexOutOfBoundsException(j);
2589                                if (clip) {
2590                                        j = 0;
2591                                } else {
2592                                        j %= n;
2593                                        j += n; // as remainder still negative
2594                                }
2595                        }
2596                        if (j >= n) {
2597                                if (throwAIOOBE)
2598                                        throw new ArrayIndexOutOfBoundsException(j);
2599                                if (clip) {
2600                                        j = n - 1;
2601                                } else {
2602                                        j %= n;
2603                                }
2604                        }
2605                        Dataset c = dChoices[j];
2606                        r.setObjectAbs(i++, c.getObject(pos));
2607                }
2608                return r;
2609        }
2610
2611        /**
2612         * Calculate positions in given shape from a dataset of 1-D indexes
2613         * @param indices
2614         * @param shape
2615         * @return list of positions as integer datasets
2616         */
2617        public static List<IntegerDataset> calcPositionsFromIndexes(Dataset indices, int[] shape) {
2618                int rank = shape.length;
2619                List<IntegerDataset> posns = new ArrayList<IntegerDataset>();
2620                int[] iShape = indices.getShapeRef();
2621                for (int i = 0; i < rank; i++) {
2622                        posns.add(new IntegerDataset(iShape));
2623                }
2624                IndexIterator it = indices.getIterator(true);
2625                int[] pos = it.getPos();
2626                while (it.hasNext()) {
2627                        int n = indices.getInt(pos);
2628                        int[] p = ShapeUtils.getNDPositionFromShape(n, shape);
2629                        for (int i = 0; i < rank; i++) {
2630                                posns.get(i).setItem(p[i], pos);
2631                        }
2632                }
2633                return posns;
2634        }
2635
2636
2637        /**
2638         * Calculate indexes in given shape from datasets of position
2639         * @param positions as a list of datasets where each holds the position in a dimension
2640         * @param shape
2641         * @param mode either null, zero-length, unit length or length of rank of shape where
2642         *  0 = raise exception, 1 = wrap, 2 = clip
2643         * @return indexes as an integer dataset
2644         */
2645        public static IntegerDataset calcIndexesFromPositions(List<? extends Dataset> positions, int[] shape, int... mode) {
2646                int rank = shape.length;
2647                if (positions.size() != rank) {
2648                        throw new IllegalArgumentException("Number of position datasets must be equal to rank of shape");
2649                }
2650
2651                if (mode == null || mode.length == 0) {
2652                        mode = new int[rank];
2653                } else if (mode.length == 1) {
2654                        int m = mode[0];
2655                        mode = new int[rank];
2656                        Arrays.fill(mode, m);
2657                } else if (mode.length != rank) {
2658                        throw new IllegalArgumentException("Mode length greater than one must match rank of shape");
2659                }
2660                for (int i = 0; i < rank; i++) {
2661                        int m = mode[i];
2662                        if (m < 0 || m > 2) {
2663                                throw new IllegalArgumentException("Unknown mode value - it must be 0, 1, or 2");
2664                        }
2665                }
2666
2667                Dataset p = positions.get(0);
2668                IntegerDataset indexes = new IntegerDataset(p.getShapeRef());
2669                IndexIterator it = p.getIterator(true);
2670                int[] iPos = it.getPos();
2671                int[] tPos = new int[rank];
2672                while (it.hasNext()) {
2673                        for (int i = 0; i < rank; i++) {
2674                                p = positions.get(i);
2675                                int j = p.getInt(iPos);
2676                                int d = shape[i];
2677                                if (mode[i] == 0) {
2678                                        if (j < 0 || j >= d) {
2679                                                throw new ArrayIndexOutOfBoundsException("Position value exceeds dimension in shape");
2680                                        }
2681                                } else if (mode[i] == 1) {
2682                                        while (j < 0)
2683                                                j += d;
2684                                        while (j >= d)
2685                                                j -= d;
2686                                } else {
2687                                        if (j < 0)
2688                                                j = 0;
2689                                        if (j >= d)
2690                                                j = d - 1;
2691                                }
2692                                tPos[i] = j;
2693                        }
2694                        indexes.set(ShapeUtils.getFlat1DIndex(shape, tPos), iPos);
2695                }
2696
2697                return indexes;
2698        }
2699
2700        /**
2701         * Serialize dataset by flattening it. Discards metadata
2702         * @param data
2703         * @return some java array
2704         */
2705        public static Serializable serializeDataset(final IDataset data) {
2706                Dataset d = convertToDataset(data).getView(false);
2707                d.clearMetadata(null);
2708                return d.flatten().getBuffer();
2709        }
2710
2711        /**
2712         * Extract values where condition is non-zero. This is similar to Dataset#getByBoolean but supports broadcasting
2713         * @param data
2714         * @param condition should be broadcastable to data
2715         * @return 1-D dataset of values
2716         */
2717        public static Dataset extract(final IDataset data, final IDataset condition) {
2718                Dataset a = convertToDataset(data.getSliceView());
2719                Dataset b = cast(BooleanDataset.class, condition.getSliceView());
2720
2721                try {
2722                        return a.getByBoolean(b);
2723                } catch (IllegalArgumentException e) {
2724                        final int length = ((Number) b.sum()).intValue();
2725
2726                        BroadcastPairIterator it = new BroadcastPairIterator(a, b, null, false);
2727                        int size = ShapeUtils.calcSize(it.getShape());
2728                        Dataset c;
2729                        if (length < size) {
2730                                int[] ashape = it.getFirstShape();
2731                                int[] bshape = it.getSecondShape();
2732                                int r = ashape.length;
2733                                size = length;
2734                                for (int i = 0; i < r; i++) {
2735                                        int s = ashape[i];
2736                                        if (s > 1 && bshape[i] == 1) {
2737                                                size *= s;
2738                                        }
2739                                }
2740                        }
2741                        c = DatasetFactory.zeros(a.getClass(), size);
2742
2743                        int i = 0;
2744                        if (it.isOutputDouble()) {
2745                                while (it.hasNext()) {
2746                                        if (it.bLong != 0) {
2747                                                c.setObjectAbs(i++, it.aDouble);
2748                                        }
2749                                }
2750                        } else {
2751                                while (it.hasNext()) {
2752                                        if (it.bLong != 0) {
2753                                                c.setObjectAbs(i++, it.aLong);
2754                                        }
2755                                }
2756                        }
2757
2758                        return c;
2759                }
2760        }
2761
2762        /**
2763         * Set shape to keep original rank
2764         * @param a
2765         * @param originalShape
2766         * @param axes
2767         * @since 2.2
2768         */
2769        public static void setShapeToOriginalRank(ILazyDataset a, int[] originalShape, int... axes) {
2770                a.setShape(ShapeUtils.getReducedShapeKeepRank(originalShape, axes));
2771        }
2772}