001/***************************************************************************** 002 * Copyright by The HDF Group. * 003 * Copyright by the Board of Trustees of the University of Illinois. * 004 * All rights reserved. * 005 * * 006 * This file is part of the HDF Java Products distribution. * 007 * The full copyright notice, including terms governing use, modification, * 008 * and redistribution, is contained in the COPYING file, which can be found * 009 * at the root of the source code distribution tree, * 010 * or in https://www.hdfgroup.org/licenses. * 011 * If you do not have access to either file, you may request a copy from * 012 * help@hdfgroup.org. * 013 ****************************************************************************/ 014 015package hdf.object; 016 017import java.lang.reflect.Array; 018import java.math.BigDecimal; 019import java.math.BigInteger; 020import java.text.DecimalFormat; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Iterator; 024import java.util.List; 025import java.util.Vector; 026 027import org.slf4j.Logger; 028import org.slf4j.LoggerFactory; 029 030import hdf.hdf5lib.HDFNativeData; 031 032import hdf.object.Attribute; 033import hdf.object.CompoundDS; 034import hdf.object.Dataset; 035import hdf.object.Datatype; 036import hdf.object.FileFormat; 037import hdf.object.Group; 038import hdf.object.HObject; 039import hdf.object.Utils; 040 041/** 042 * A CompoundDS is a dataset with compound datatype. 043 * 044 * A compound datatype is an aggregation of one or more datatypes. Each member of a compound type has a name which is 045 * unique within that type, and a datatype of that member in a compound datum. Compound datatypes can be nested, i.e. 046 * members of a compound datatype can be some other compound datatype. 047 * 048 * For more details on compound datatypes, See 049 * <a href="https://hdfgroup.github.io/hdf5/_h5_t__u_g.html#sec_datatype">HDF5 Datatypes in HDF5 User Guide</a> 050 * 051 * Since Java cannot handle C-structured compound data, data in a compound dataset is loaded in to an Java List. Each 052 * element of the list is a data array that corresponds to a compound field. The data is read/written by compound field. 053 * 054 * For example, if compound dataset "comp" has the following nested structure, and member datatypes 055 * 056 * <pre> 057 * comp --> m01 (int) 058 * comp --> m02 (float) 059 * comp --> nest1 --> m11 (char) 060 * comp --> nest1 --> m12 (String) 061 * comp --> nest1 --> nest2 --> m21 (long) 062 * comp --> nest1 --> nest2 --> m22 (double) 063 * </pre> 064 * 065 * The data object is a Java list of six arrays: {int[], float[], char[], Stirng[], long[] and double[]}. 066 * 067 * 068 * @version 1.1 9/4/2007 069 * @author Peter X. Cao 070 */ 071public abstract class CompoundDS extends Dataset implements CompoundDataFormat 072{ 073 private static final long serialVersionUID = -4880399929644095662L; 074 075 private static final Logger log = LoggerFactory.getLogger(CompoundDS.class); 076 077 /** 078 * A single character to separate the names of nested compound fields. An 079 * extended ASCII character, 0x95, is used to avoid common characters in 080 * compound names. 081 */ 082 public static final String SEPARATOR = "\u0095"; 083 084 /** 085 * The number of members of the compound dataset. 086 */ 087 protected int numberOfMembers; 088 089 /** 090 * The names of members of the compound dataset. 091 */ 092 protected String[] memberNames; 093 094 /** 095 * Returns array containing the total number of elements of the members of 096 * this compound dataset. 097 * 098 * For example, a compound dataset COMP has members of A, B and C as 099 * 100 * <pre> 101 * COMP { 102 * int A; 103 * float B[5]; 104 * double C[2][3]; 105 * } 106 * </pre> 107 * 108 * memberOrders is an integer array of {1, 5, 6} to indicate that member A 109 * has one element, member B has 5 elements, and member C has 6 elements. 110 */ 111 protected int[] memberOrders; 112 113 /** 114 * The dimension sizes of each member. 115 * 116 * The i-th element of the Object[] is an integer array (int[]) that 117 * contains the dimension sizes of the i-th member. 118 */ 119 protected transient Object[] memberDims; 120 121 /** 122 * The datatypes of compound members. 123 */ 124 protected Datatype[] memberTypes; 125 126 /** 127 * The array to store flags to indicate if a member of this compound 128 * dataset is selected for read/write. 129 * 130 * If a member is selected, the read/write will perform on the member. 131 * Applications such as HDFView will only display the selected members of 132 * the compound dataset. 133 * 134 * <pre> 135 * For example, if a compound dataset has four members 136 * String[] memberNames = {"X", "Y", "Z", "TIME"}; 137 * and 138 * boolean[] isMemberSelected = {true, false, false, true}; 139 * members "X" and "TIME" are selected for read and write. 140 * </pre> 141 */ 142 protected boolean[] isMemberSelected; 143 144 /** 145 * A list of names of all fields including nested fields. 146 * 147 * The nested names are separated by CompoundDS.SEPARATOR. For example, if compound dataset "A" has 148 * the following nested structure, 149 * 150 * <pre> 151 * A --> m01 152 * A --> m02 153 * A --> nest1 --> m11 154 * A --> nest1 --> m12 155 * A --> nest1 --> nest2 --> m21 156 * A --> nest1 --> nest2 --> m22 157 * i.e. 158 * A = { m01, m02, nest1{m11, m12, nest2{ m21, m22}}} 159 * </pre> 160 * 161 * The flatNameList of compound dataset "A" will be {m01, m02, nest1[m11, nest1[m12, 162 * nest1[nest2[m21, nest1[nest2[m22} 163 * 164 */ 165 protected List<String> flatNameList; 166 167 /** 168 * A list of datatypes of all fields including nested fields. 169 */ 170 protected List<Datatype> flatTypeList; 171 172 /** 173 * Constructs a CompoundDS object with the given file, dataset name and path. 174 * 175 * The dataset object represents an existing dataset in the file. For 176 * example, new CompoundDS(file, "dset1", "/g0/") constructs a dataset 177 * object that corresponds to the dataset, "dset1", at group "/g0/". 178 * 179 * This object is usually constructed at FileFormat.open(), which loads the 180 * file structure and object information into memory. It is rarely used 181 * elsewhere. 182 * 183 * @param theFile 184 * the file that contains the data object. 185 * @param theName 186 * the name of the data object, e.g. "dset". 187 * @param thePath 188 * the full path of the data object, e.g. "/arrays/". 189 */ 190 public CompoundDS(FileFormat theFile, String theName, String thePath) { 191 this(theFile, theName, thePath, null); 192 } 193 194 /** 195 * @deprecated Not for public use in the future.<br> 196 * Using {@link #CompoundDS(FileFormat, String, String)} 197 * 198 * @param theFile 199 * the file that contains the data object. 200 * @param dsName 201 * the name of the data object, e.g. "dset". 202 * @param dsPath 203 * the full path of the data object, e.g. "/arrays/". 204 * @param oid 205 * the oid of the data object. 206 */ 207 @Deprecated 208 public CompoundDS(FileFormat theFile, String dsName, String dsPath, long[] oid) { 209 super(theFile, dsName, dsPath, oid); 210 211 numberOfMembers = 0; 212 memberNames = null; 213 isMemberSelected = null; 214 memberTypes = null; 215 } 216 217 /** 218 * Resets selection of dataspace 219 */ 220 @Override 221 protected void resetSelection() { 222 super.resetSelection(); 223 setAllMemberSelection(true); 224 } 225 226 /** 227 * Returns the number of members of the compound dataset. 228 * 229 * @return the number of members of the compound dataset. 230 */ 231 @Override 232 public final int getMemberCount() { 233 return numberOfMembers; 234 } 235 236 /** 237 * Returns the number of selected members of the compound dataset. 238 * 239 * Selected members are the compound fields which are selected for 240 * read/write. 241 * 242 * For example, in a compound datatype of {int A, float B, char[] C}, 243 * users can choose to retrieve only {A, C} from the dataset. In this 244 * case, getSelectedMemberCount() returns two. 245 * 246 * @return the number of selected members. 247 */ 248 @Override 249 public final int getSelectedMemberCount() { 250 int count = 0; 251 252 if (isMemberSelected != null) { 253 for (int i = 0; i < isMemberSelected.length; i++) { 254 if (isMemberSelected[i]) 255 count++; 256 } 257 } 258 log.trace("count of selected members={}", count); 259 260 return count; 261 } 262 263 /** 264 * Returns the names of the members of the compound dataset. The names of 265 * compound members are stored in an array of Strings. 266 * 267 * For example, for a compound datatype of {int A, float B, char[] C} 268 * getMemberNames() returns ["A", "B", "C"}. 269 * 270 * @return the names of compound members. 271 */ 272 @Override 273 public final String[] getMemberNames() { 274 return memberNames; 275 } 276 277 /** 278 * Returns an array of the names of the selected members of the compound dataset. 279 * 280 * @return an array of the names of the selected members of the compound dataset. 281 */ 282 @Override 283 public final String[] getSelectedMemberNames() { 284 if (isMemberSelected == null) { 285 log.debug("getSelectedMemberNames(): isMemberSelected array is null"); 286 return memberNames; 287 } 288 289 int idx = 0; 290 String[] names = new String[getSelectedMemberCount()]; 291 for (int i = 0; i < isMemberSelected.length; i++) { 292 if (isMemberSelected[i]) 293 names[idx++] = memberNames[i]; 294 } 295 296 return names; 297 } 298 299 /** 300 * Checks if a member of the compound dataset is selected for read/write. 301 * 302 * @param idx 303 * the index of compound member. 304 * 305 * @return true if the i-th memeber is selected; otherwise returns false. 306 */ 307 @Override 308 public final boolean isMemberSelected(int idx) { 309 if ((isMemberSelected != null) && (isMemberSelected.length > idx)) 310 return isMemberSelected[idx]; 311 else 312 return false; 313 } 314 315 /** 316 * Selects the i-th member for read/write. 317 * 318 * @param idx 319 * the index of compound member. 320 */ 321 @Override 322 public final void selectMember(int idx) { 323 if ((isMemberSelected != null) && (isMemberSelected.length > idx)) 324 isMemberSelected[idx] = true; 325 } 326 327 /** 328 * Selects/deselects all members. 329 * 330 * @param selectAll 331 * The indicator to select or deselect all members. If true, all 332 * members are selected for read/write. If false, no member is 333 * selected for read/write. 334 */ 335 @Override 336 public final void setAllMemberSelection(boolean selectAll) { 337 if (isMemberSelected == null) 338 return; 339 340 for (int i = 0; i < isMemberSelected.length; i++) 341 isMemberSelected[i] = selectAll; 342 } 343 344 /** 345 * Returns array containing the total number of elements of the members of 346 * the compound dataset. 347 * 348 * For example, a compound dataset COMP has members of A, B and C as 349 * 350 * <pre> 351 * COMP { 352 * int A; 353 * float B[5]; 354 * double C[2][3]; 355 * } 356 * </pre> 357 * 358 * getMemberOrders() will return an integer array of {1, 5, 6} to indicate 359 * that member A has one element, member B has 5 elements, and member C has 360 * 6 elements. 361 * 362 * @return the array containing the total number of elements of the members 363 * of compound. 364 */ 365 @Override 366 public final int[] getMemberOrders() { 367 return memberOrders; 368 } 369 370 /** 371 * Returns array containing the total number of elements of the selected 372 * members of the compound dataset. 373 * 374 * For example, a compound dataset COMP has members of A, B and C as 375 * 376 * <pre> 377 * COMP { 378 * int A; 379 * float B[5]; 380 * double C[2][3]; 381 * } 382 * </pre> 383 * 384 * If A and B are selected, getSelectedMemberOrders() returns an array of 385 * {1, 5} 386 * 387 * @return array containing the total number of elements of the selected 388 * members of compound. 389 */ 390 @Override 391 public final int[] getSelectedMemberOrders() { 392 if (isMemberSelected == null) { 393 log.debug("getSelectedMemberOrders(): isMemberSelected array is null"); 394 return memberOrders; 395 } 396 397 int idx = 0; 398 int[] orders = new int[getSelectedMemberCount()]; 399 for (int i = 0; i < isMemberSelected.length; i++) { 400 if (isMemberSelected[i]) 401 orders[idx++] = memberOrders[i]; 402 } 403 404 return orders; 405 } 406 407 /** 408 * Returns the dimension sizes of the i-th member. 409 * 410 * For example, a compound dataset COMP has members of A, B and C as 411 * 412 * <pre> 413 * COMP { 414 * int A; 415 * float B[5]; 416 * double C[2][3]; 417 * } 418 * </pre> 419 * 420 * getMemberDims(2) returns an array of {2, 3}, while getMemberDims(1) 421 * returns an array of {5}, and getMemberDims(0) returns null. 422 * 423 * @param i the i-th member 424 * 425 * @return the dimension sizes of the i-th member, null if the compound 426 * member is not an array. 427 */ 428 @Override 429 public final int[] getMemberDims(int i) { 430 if (memberDims == null) { 431 return null; 432 } 433 return (int[]) memberDims[i]; 434 } 435 436 /** 437 * Returns an array of datatype objects of compound members. 438 * 439 * Each member of a compound dataset has its own datatype. The datatype of a 440 * member can be atomic or other compound datatype (nested compound). 441 * Sub-classes set up the datatype objects at init(). 442 * 443 * @return the array of datatype objects of the compound members. 444 */ 445 @Override 446 public final Datatype[] getMemberTypes() { 447 return memberTypes; 448 } 449 450 /** 451 * Returns an array of datatype objects of selected compound members. 452 * 453 * @return an array of datatype objects of selected compound members. 454 */ 455 @Override 456 public final Datatype[] getSelectedMemberTypes() { 457 if (isMemberSelected == null) { 458 log.debug("getSelectedMemberTypes(): isMemberSelected array is null"); 459 return memberTypes; 460 } 461 462 int idx = 0; 463 Datatype[] types = new Datatype[getSelectedMemberCount()]; 464 for (int i = 0; i < isMemberSelected.length; i++) { 465 if (isMemberSelected[i]) 466 types[idx++] = memberTypes[i]; 467 } 468 469 return types; 470 } 471 472 /** 473 * Returns the fill values for the data object. 474 * 475 * @return the fill values for the data object. 476 */ 477 @Override 478 public Object getFillValue() { 479 return null; 480 } 481 482 /** 483 * @deprecated Not implemented for compound dataset. 484 */ 485 @Deprecated 486 @Override 487 public Dataset copy(Group pgroup, String name, long[] dims, Object data) throws Exception { 488 throw new UnsupportedOperationException( 489 "Writing a subset of a compound dataset to a new dataset is not implemented."); 490 } 491 492 /** 493 * Routine to convert datatypes that are read in as byte arrays to 494 * regular types. 495 * 496 * @param dtype 497 * the datatype to convert to 498 * @param byteData 499 * the bytes to convert 500 * 501 * @return the converted object 502 */ 503 protected Object convertByteMember(final Datatype dtype, byte[] byteData) { 504 Object theObj = null; 505 log.trace("convertByteMember(): byteData={} start", byteData); 506 507 if (dtype.getDatatypeSize() == 1) { 508 /* 509 * Normal byte[] type, such as an integer datatype of size 1. 510 */ 511 theObj = byteData; 512 } 513 else if (dtype.isString() && !dtype.isVarStr() && convertByteToString && (byteData instanceof byte[])) { 514 log.trace("convertByteMember(): converting byte array to string array"); 515 516 theObj = byteToString(byteData, (int) dtype.getDatatypeSize()); 517 } 518 else if (dtype.isInteger()) { 519 log.trace("convertByteMember(): converting byte array to integer array"); 520 521 switch ((int)dtype.getDatatypeSize()) { 522 case 1: 523 /* 524 * Normal byte[] type, such as an integer datatype of size 1. 525 */ 526 theObj = byteData; 527 break; 528 case 2: 529 theObj = HDFNativeData.byteToShort(byteData); 530 break; 531 case 4: 532 theObj = HDFNativeData.byteToInt(byteData); 533 break; 534 case 8: 535 theObj = HDFNativeData.byteToLong(byteData); 536 break; 537 default: 538 log.debug("convertByteMember(): invalid datatype size"); 539 theObj = new String("*ERROR*"); 540 break; 541 } 542 } 543 else if (dtype.isFloat()) { 544 log.trace("convertByteMember(): converting byte array to float array"); 545 546 if (dtype.getDatatypeSize() == 8) 547 theObj = HDFNativeData.byteToDouble(byteData); 548 else 549 theObj = HDFNativeData.byteToFloat(byteData); 550 } 551 else if (dtype.isArray()) { 552 Datatype baseType = dtype.getDatatypeBase(); 553 log.trace("convertByteMember(): converting byte array to baseType array"); 554 555 /* 556 * Retrieve the real base datatype in the case of ARRAY of ARRAY datatypes. 557 */ 558 while (baseType.isArray()) 559 baseType = baseType.getDatatypeBase(); 560 561 /* 562 * Optimize for the common cases of Arrays. 563 */ 564 switch (baseType.getDatatypeClass()) { 565 case Datatype.CLASS_INTEGER: 566 case Datatype.CLASS_FLOAT: 567 case Datatype.CLASS_CHAR: 568 case Datatype.CLASS_STRING: 569 case Datatype.CLASS_BITFIELD: 570 case Datatype.CLASS_OPAQUE: 571 case Datatype.CLASS_COMPOUND: 572 case Datatype.CLASS_REFERENCE: 573 case Datatype.CLASS_ENUM: 574 case Datatype.CLASS_VLEN: 575 case Datatype.CLASS_TIME: 576 theObj = convertByteMember(baseType, byteData); 577 break; 578 579 case Datatype.CLASS_ARRAY: 580 { 581 Datatype arrayType = dtype.getDatatypeBase(); 582 583 long[] arrayDims = dtype.getArrayDims(); 584 int arrSize = 1; 585 for (int i = 0; i < arrayDims.length; i++) 586 arrSize *= arrayDims[i]; 587 log.trace("convertByteMember(): no CLASS_ARRAY arrayType={} arrSize={}", arrayType, arrSize); 588 589 theObj = new Object[arrSize]; 590 591 for (int i = 0; i < arrSize; i++) { 592 byte[] indexedBytes = Arrays.copyOfRange(byteData, (int) (i * arrayType.getDatatypeSize()), 593 (int) ((i + 1) * arrayType.getDatatypeSize())); 594 ((Object[]) theObj)[i] = convertByteMember(arrayType, indexedBytes); 595 } 596 597 break; 598 } 599 600 case Datatype.CLASS_NO_CLASS: 601 default: 602 log.debug("convertByteMember(): invalid datatype class"); 603 theObj = new String("*ERROR*"); 604 } 605 } 606 else if (dtype.isCompound()) { 607 log.debug("convertByteMember(): compound datatype class"); 608 /* 609 * TODO: still valid after reading change? 610 */ 611 theObj = convertCompoundByteMembers(dtype, byteData); 612 } 613 else { 614 log.debug("convertByteMember(): byteData={}", byteData); 615 theObj = byteData; 616 } 617 618 return theObj; 619 } 620 621 /** 622 * Given an array of bytes representing a compound Datatype, converts each of 623 * its members into Objects and returns the results. 624 * 625 * @param dtype 626 * The compound datatype to convert 627 * @param data 628 * The byte array representing the data of the compound Datatype 629 * @return The converted types of the bytes 630 */ 631 protected Object convertCompoundByteMembers(final Datatype dtype, byte[] data) { 632 List<Object> theData = null; 633 634 List<Datatype> allSelectedTypes = Arrays.asList(this.getSelectedMemberTypes()); 635 List<Datatype> localTypes = new ArrayList<>(dtype.getCompoundMemberTypes()); 636 Iterator<Datatype> localIt = localTypes.iterator(); 637 while (localIt.hasNext()) { 638 Datatype curType = localIt.next(); 639 640 if (curType.isCompound()) 641 continue; 642 643 if (!allSelectedTypes.contains(curType)) 644 localIt.remove(); 645 } 646 647 theData = new ArrayList<>(localTypes.size()); 648 for (int i = 0, index = 0; i < localTypes.size(); i++) { 649 Datatype curType = localTypes.get(i); 650 651 if (curType.isCompound()) 652 theData.add(convertCompoundByteMembers(curType, 653 Arrays.copyOfRange(data, index, index + (int) curType.getDatatypeSize()))); 654 else 655 theData.add(convertByteMember(curType, 656 Arrays.copyOfRange(data, index, index + (int) curType.getDatatypeSize()))); 657 658 index += curType.getDatatypeSize(); 659 } 660 661 return theData; 662 } 663}