001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.IOException; 016import java.io.Serializable; 017import java.lang.annotation.Annotation; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.HashMap; 022import java.util.List; 023import java.util.Map; 024 025import org.eclipse.january.DatasetException; 026import org.eclipse.january.IMonitor; 027import org.eclipse.january.io.ILazyLoader; 028import org.eclipse.january.metadata.MetadataFactory; 029import org.eclipse.january.metadata.MetadataType; 030import org.eclipse.january.metadata.OriginMetadata; 031import org.eclipse.january.metadata.Reshapeable; 032import org.eclipse.january.metadata.Sliceable; 033import org.eclipse.january.metadata.Transposable; 034 035public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable { 036 private static final long serialVersionUID = 2467865859867440242L; 037 038 protected int[] oShape; // original shape 039 protected long size; // number of items 040 protected int dtype; // dataset type 041 protected int isize; // number of elements per item 042 043 protected ILazyLoader loader; 044 protected LazyDataset base = null; // used for transpose 045 046 // relative to loader or base 047 protected int prepShape = 0; // prepending and post-pending 048 protected int postShape = 0; // changes to shape 049 protected int[] begSlice = null; // slice begin 050 protected int[] delSlice = null; // slice delta 051 protected int[] map; // transposition map (same length as current shape) 052 protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null; 053 054 /** 055 * Create a lazy dataset 056 * @param name 057 * @param dtype dataset type 058 * @param elements 059 * @param shape 060 * @param loader 061 */ 062 public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) { 063 this.name = name; 064 this.shape = shape.clone(); 065 this.oShape = this.shape; 066 this.loader = loader; 067 this.dtype = dtype; 068 this.isize = elements; 069 try { 070 size = ShapeUtils.calcLongSize(shape); 071 } catch (IllegalArgumentException e) { 072 size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 073 } 074 } 075 076 /** 077 * Create a lazy dataset 078 * @param name 079 * @param dtype dataset type 080 * @param shape 081 * @param loader 082 */ 083 public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) { 084 this(name, dtype, 1, shape, loader); 085 } 086 087 /** 088 * Create a lazy dataset based on in-memory data (handy for testing) 089 * @param dataset 090 */ 091 public static LazyDataset createLazyDataset(final Dataset dataset) { 092 return new LazyDataset(dataset.getName(), dataset.getDType(), dataset.getElementsPerItem(), dataset.getShape(), 093 new ILazyLoader() { 094 private static final long serialVersionUID = -6725268922780517523L; 095 096 final Dataset d = dataset; 097 @Override 098 public boolean isFileReadable() { 099 return true; 100 } 101 102 @Override 103 public Dataset getDataset(IMonitor mon, SliceND slice) throws IOException { 104 return d.getSlice(mon, slice); 105 } 106 }); 107 } 108 109 /** 110 * Can return -1 for unknown 111 */ 112 @Override 113 public int getDType() { 114 return dtype; 115 } 116 117 /** 118 * Can return -1 for unknown 119 */ 120 @Override 121 public int getElementsPerItem() { 122 return isize; 123 } 124 125 @Override 126 public int getSize() { 127 return (int) size; 128 } 129 130 @Override 131 public String toString() { 132 StringBuilder out = new StringBuilder(); 133 134 if (name != null && name.length() > 0) { 135 out.append("Lazy dataset '"); 136 out.append(name); 137 out.append("' has shape ["); 138 } else { 139 out.append("Lazy dataset shape is ["); 140 } 141 int rank = shape == null ? 0 : shape.length; 142 143 if (rank > 0 && shape[0] >= 0) { 144 out.append(shape[0]); 145 } 146 for (int i = 1; i < rank; i++) { 147 out.append(", " + shape[i]); 148 } 149 out.append(']'); 150 151 return out.toString(); 152 } 153 154 @Override 155 public boolean equals(Object obj) { 156 if (!super.equals(obj)) 157 return false; 158 159 LazyDataset other = (LazyDataset) obj; 160 if (dtype != other.dtype) { 161 return false; 162 } 163 if (isize != other.isize) { 164 return false; 165 } 166 167 if (!Arrays.equals(shape, other.shape)) { 168 return false; 169 } 170 171 if (loader != other.loader) { 172 return false; 173 } 174 175 if (prepShape != other.prepShape) { 176 return false; 177 } 178 179 if (postShape != other.postShape) { 180 return false; 181 } 182 183 if (!Arrays.equals(begSlice, other.begSlice)) { 184 return false; 185 } 186 if (!Arrays.equals(delSlice, other.delSlice)) { 187 return false; 188 } 189 if (!Arrays.equals(map, other.map)) { 190 return false; 191 } 192 return true; 193 } 194 195 @Override 196 public LazyDataset clone() { 197 LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader); 198 ret.shape = shape; 199 ret.size = size; 200 ret.prepShape = prepShape; 201 ret.postShape = postShape; 202 ret.begSlice = begSlice; 203 ret.delSlice = delSlice; 204 ret.map = map; 205 ret.base = base; 206 ret.metadata = copyMetadata(); 207 ret.oMetadata = oMetadata; 208 return ret; 209 } 210 211 @Override 212 public void setShape(int... shape) { 213 setShapeInternal(shape); 214 } 215 216 @Override 217 public LazyDataset squeezeEnds() { 218 setShapeInternal(ShapeUtils.squeezeShape(shape, true)); 219 return this; 220 } 221 222 @Override 223 public Dataset getSlice(int[] start, int[] stop, int[] step) throws DatasetException { 224 return getSlice(null, start, stop, step); 225 } 226 227 @Override 228 public Dataset getSlice(Slice... slice) throws DatasetException { 229 if (slice == null || slice.length == 0) { 230 return getSlice(null, new SliceND(shape)); 231 } 232 return getSlice(null, new SliceND(shape, slice)); 233 } 234 235 @Override 236 public Dataset getSlice(SliceND slice) throws DatasetException { 237 return getSlice(null, slice); 238 } 239 240 @Override 241 public Dataset getSlice(IMonitor monitor, Slice... slice) throws DatasetException { 242 if (slice == null || slice.length == 0) { 243 return getSlice(monitor, new SliceND(shape)); 244 } 245 return getSlice(monitor, new SliceND(shape, slice)); 246 } 247 248 @Override 249 public LazyDataset getSliceView(Slice... slice) { 250 if (slice == null || slice.length == 0) { 251 return getSliceView(new SliceND(shape)); 252 } 253 return getSliceView(new SliceND(shape, slice)); 254 } 255 256 /** 257 * @param nShape 258 */ 259 private void setShapeInternal(int... nShape) { 260 261 long nsize = ShapeUtils.calcLongSize(nShape); 262 if (nsize != size) { 263 throw new IllegalArgumentException("Size of new shape is not equal to current size"); 264 } 265 266 if (nsize == 1) { 267 shape = nShape.clone(); 268 return; 269 } 270 271 int ob = -1; // first non-unit dimension 272 int or = shape.length; 273 for (int i = 0; i < or; i++) { 274 if (shape[i] != 1) { 275 ob = i; 276 break; 277 } 278 } 279 assert ob >= 0; 280 int oe = -1; // last non-unit dimension 281 for (int i = or - 1; i >= ob; i--) { 282 if (shape[i] != 1) { 283 oe = i; 284 break; 285 } 286 } 287 assert oe >= 0; 288 oe++; 289 290 int nb = -1; // first non-unit dimension 291 int nr = nShape.length; 292 for (int i = 0; i < nr; i++) { 293 if (nShape[i] != 1) { 294 nb = i; 295 break; 296 } 297 } 298 299 int i = ob; 300 int j = nb; 301 if (begSlice == null) { 302 for (; i < oe && j < nr; i++, j++) { 303 if (shape[i] != nShape[j]) { 304 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 305 } 306 } 307 } else { 308 int[] nBegSlice = new int[nr]; 309 int[] nDelSlice = new int[nr]; 310 Arrays.fill(nDelSlice, 1); 311 for (; i < oe && j < nr; i++, j++) { 312 if (shape[i] != nShape[j]) { 313 throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape"); 314 } 315 nBegSlice[j] = begSlice[i]; 316 nDelSlice[j] = delSlice[i]; 317 } 318 319 begSlice = nBegSlice; 320 delSlice = nDelSlice; 321 } 322 prepShape += nb - ob; 323 postShape += nr - oe; 324 325 storeMetadata(metadata, Reshapeable.class); 326 metadata = copyMetadata(); 327 reshapeMetadata(shape, nShape); 328 shape = nShape; 329 } 330 331 @Override 332 public LazyDataset getSliceView(int[] start, int[] stop, int[] step) { 333 return getSliceView(new SliceND(shape, start, stop, step)); 334 } 335 336 @Override 337 public LazyDataset getSliceView(SliceND slice) { 338 LazyDataset view = clone(); 339 if (slice.isAll()) 340 return view; 341 342 int[] lstart = slice.getStart(); 343 int[] lstep = slice.getStep(); 344 final int rank = shape.length; 345 346 int[] nShape = slice.getShape(); 347 view.shape = nShape; 348 view.size = ShapeUtils.calcLongSize(nShape); 349 if (begSlice == null) { 350 view.begSlice = lstart.clone(); 351 view.delSlice = lstep.clone(); 352 } else { 353 view.begSlice = new int[rank]; 354 view.delSlice = new int[rank]; 355 for (int i = 0; i < rank; i++) { 356 view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i]; 357 view.delSlice[i] = delSlice[i] * lstep[i]; 358 } 359 } 360 view.storeMetadata(metadata, Sliceable.class); 361 362 view.sliceMetadata(true, slice); 363 return view; 364 } 365 366 @Override 367 public LazyDataset getTransposedView(int... axes) { 368 LazyDataset view = clone(); 369 370 // everything now is seen through a map 371 axes = checkPermutatedAxes(shape, axes); 372 if (axes == null) 373 return view; 374 375 int r = shape.length; 376 view.shape = new int[r]; 377 for (int i = 0; i < r; i++) { 378 view.shape[i] = shape[axes[i]]; 379 } 380 381 view.prepShape = 0; 382 view.postShape = 0; 383 view.begSlice = null; 384 view.delSlice = null; 385 view.map = axes; 386 view.base = this; 387 view.storeMetadata(metadata, Transposable.class); 388 view.transposeMetadata(axes); 389 return view; 390 } 391 392 @Override 393 public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws DatasetException { 394 return getSlice(monitor, new SliceND(shape, start, stop, step)); 395 } 396 397 @Override 398 public Dataset getSlice(IMonitor monitor, SliceND slice) throws DatasetException { 399 400 if (loader != null && !loader.isFileReadable()) 401 return null; // TODO add interaction to use plot (or remote) server to load dataset 402 403 SliceND nslice = calcTrueSlice(slice); 404 405 Dataset a; 406 if (base != null) { 407 a = base.getSlice(monitor, nslice); 408 } else { 409 try { 410 a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice)); 411 } catch (IOException e) { 412 logger.error("Problem getting {}: {}", String.format("slice %s %s %s from %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()), 413 Arrays.toString(slice.getStep()), loader), e); 414 throw new DatasetException(e); 415 } 416 a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE); 417 if (metadata != null && a instanceof LazyDatasetBase) { 418 LazyDatasetBase ba = (LazyDatasetBase) a; 419 ba.metadata = copyMetadata(); 420 if (oMetadata != null) 421 ba.restoreMetadata(oMetadata); 422 //metadata axis may be larger than data 423 if (!nslice.isAll() || nslice.getMaxShape() != nslice.getShape()) 424 ba.sliceMetadata(true, nslice); 425 } 426 } 427 if (map != null) { 428 a = a.getTransposedView(map); 429 } 430 if (slice != null) 431 a.setShape(slice.getShape()); 432 a.addMetadata(MetadataFactory.createMetadata(OriginMetadata.class, this, nslice.convertToSlice(), oShape, null, name)); 433 434 return a; 435 } 436 437 // reverse transform 438 private int[] getOriginal(int[] values) { 439 if (values == null) 440 return null; 441 int r = values.length; 442 if (map == null || r < 2) 443 return values; 444 int[] ovalues = new int[r]; 445 for (int i = 0; i < r; i++) { 446 ovalues[map[i]] = values[i]; 447 } 448 return ovalues; 449 } 450 451 protected final SliceND calcTrueSlice(SliceND slice) { 452 if (slice == null) { 453 slice = new SliceND(shape); 454 } 455 int[] lstart = slice.getStart(); 456 int[] lstop = slice.getStop(); 457 int[] lstep = slice.getStep(); 458 459 int[] nstart; 460 int[] nstop; 461 int[] nstep; 462 463 int r = base == null ? oShape.length : base.shape.length; 464 nstart = new int[r]; 465 nstop = new int[r]; 466 nstep = new int[r]; 467 Arrays.fill(nstop, 1); 468 Arrays.fill(nstep, 1); 469 { 470 int i = 0; 471 int j = 0; 472 if (prepShape < 0) { // ignore entries from new slice 473 i = -prepShape; 474 } else if (prepShape > 0) { 475 j = prepShape; 476 } 477 if (begSlice == null) { 478 for (; i < r && j < shape.length; i++, j++) { 479 nstart[i] = lstart[j]; 480 nstop[i] = lstop[j]; 481 int d = lstep[j]; 482 if (d < 0 && nstop[i] < 0) { // need to wrap around further 483 int l = base == null ? oShape[j]: base.shape[j]; 484 nstop[i] -= l; 485 } 486 nstep[i] = d; 487 } 488 } else { 489 for (; i < r && j < shape.length; i++, j++) { 490 int b = begSlice[j]; 491 int d = delSlice[j]; 492 nstart[i] = b + lstart[j] * d; 493 nstop[i] = b + (lstop[j] - 1) * d + (d >= 0 ? 1 : -1); 494 if (d < 0 && nstop[i] < 0) { // need to wrap around further 495 int l = base == null ? oShape[j]: base.shape[j]; 496 nstop[i] -= l; 497 } 498 nstep[i] = lstep[j] * d; 499 } 500 } 501 if (map != null) { 502 nstart = getOriginal(nstart); 503 nstop = getOriginal(nstop); 504 nstep = getOriginal(nstep); 505 } 506 } 507 508 return createSlice(nstart, nstop, nstep); 509 } 510 511 protected final IDataset transformInput(IDataset data) { 512 if (map == null) 513 return data; 514 return data.getTransposedView(map); 515 } 516 517 protected SliceND createSlice(int[] nstart, int[] nstop, int[] nstep) { 518 return new SliceND(base == null ? oShape : base.shape, nstart, nstop, nstep); 519 } 520 521 /** 522 * Store metadata items that has given annotation 523 * @param origMetadata 524 * @param aclazz 525 */ 526 private void storeMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> origMetadata, Class<? extends Annotation> aclazz) { 527 List<Class<? extends MetadataType>> mclazzes = findAnnotatedMetadata(aclazz); 528 if (mclazzes.size() == 0) 529 return; 530 531 if (oMetadata == null) { 532 oMetadata = new HashMap<Class<? extends MetadataType>, List<MetadataType>>(); 533 } 534 for (Class<? extends MetadataType> mc : mclazzes) { 535 if (oMetadata.containsKey(mc)) 536 continue; // do not overwrite original 537 538 List<MetadataType> l = origMetadata.get(mc); 539 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 540 for (MetadataType m : l) { 541 nl.add(m.clone()); 542 } 543 oMetadata.put(mc, nl); 544 } 545 } 546 547 @SuppressWarnings("unchecked") 548 private List<Class<? extends MetadataType>> findAnnotatedMetadata(Class<? extends Annotation> aclazz) { 549 List<Class<? extends MetadataType>> mclazzes = new ArrayList<Class<? extends MetadataType>>(); 550 if (metadata == null) 551 return mclazzes; 552 553 for (Class<? extends MetadataType> c : metadata.keySet()) { 554 boolean hasAnn = false; 555 for (MetadataType m : metadata.get(c)) { 556 if (m == null) 557 continue; 558 559 Class<? extends MetadataType> mc = m.getClass(); 560 do { // iterate over super-classes 561 for (Field f : mc.getDeclaredFields()) { 562 if (f.isAnnotationPresent(aclazz)) { 563 hasAnn = true; 564 break; 565 } 566 } 567 Class<?> sclazz = mc.getSuperclass(); 568 if (!MetadataType.class.isAssignableFrom(sclazz)) 569 break; 570 mc = (Class<? extends MetadataType>) sclazz; 571 } while (!hasAnn); 572 if (hasAnn) 573 break; 574 } 575 if (hasAnn) { 576 mclazzes.add(c); 577 } 578 } 579 return mclazzes; 580 } 581 582 /** 583 * Gets the maximum size of a slice of a dataset in a given dimension 584 * which should normally fit in memory. Note that it might be possible 585 * to get more in memory, this is a conservative estimate and seems to 586 * almost always work at the size returned; providing Xmx is less than 587 * the physical memory. 588 * 589 * To get more in memory increase -Xmx setting or use an expression 590 * which calls a rolling function (like rmean) instead of slicing directly 591 * to memory. 592 * 593 * @param lazySet 594 * @param dimension 595 * @return maximum size of dimension that can be sliced. 596 */ 597 public static int getMaxSliceLength(ILazyDataset lazySet, int dimension) { 598 // size in bytes of each item 599 final double size = DTypeUtils.getItemBytes(DTypeUtils.getDTypeFromClass(lazySet.getElementClass()), lazySet.getElementsPerItem()); 600 601 // Max in bytes takes into account our minimum requirement 602 final double max = Math.max(Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory()); 603 604 // Firstly if the whole dataset it likely to fit in memory, then we allow it. 605 // Space specified in bytes per item available 606 final double space = max/lazySet.getSize(); 607 608 // If we have room for this whole dataset, then fine 609 int[] shape = lazySet.getShape(); 610 if (space >= size) 611 return shape[dimension]; 612 613 // Otherwise estimate what we can fit in, conservatively. 614 // First get size of one slice, see it that fits, if not, still return 1 615 double sizeOneSlice = size; // in bytes 616 for (int dim = 0; dim < shape.length; dim++) { 617 if (dim == dimension) 618 continue; 619 sizeOneSlice *= shape[dim]; 620 } 621 double avail = max / sizeOneSlice; 622 if (avail < 1) 623 return 1; 624 625 // We fudge this to leave some room 626 return (int) Math.floor(avail/4d); 627 } 628}