TestUtil.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.datasketches.common;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

/**
 * Utilities common to testing
 */
public final class TestUtil  {

  private static final String userDir = System.getProperty("user.dir");

  /**
   * TestNG group constants
   */
  public static final String GENERATE_JAVA_FILES = "generate_java_files";
  public static final String CHECK_CPP_FILES = "check_cpp_files";
  public static final String CHECK_GO_FILES = "check_go_files";
  public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files";

  /**
   * The full target Path for Java serialized sketches to be tested by other languages.
   */
  public static final Path javaPath = createPath("serialization_test_data/java_generated_files");

  /**
   * The full target Path for C++ serialized sketches to be tested by Java.
   */
  public static final Path cppPath = createPath("serialization_test_data/cpp_generated_files");

  /**
   * The full target Path for Go serialized sketches to be tested by Java.
   */
  public static final Path goPath = createPath("serialization_test_data/go_generated_files");

  private static Path createPath(final String projectLocalDir) {
    try {
      return Files.createDirectories(Paths.get(userDir, projectLocalDir));
    } catch (IOException e) { throw new SketchesArgumentException(e.getCause().toString()); }
  }

  //Get Resources

  private static final int BUF_SIZE = 1 << 13;

  /**
   * Gets the file defined by the given resource file's shortFileName.
   * @param shortFileName the last name in the pathname's name sequence.
   * @return the file defined by the given resource file's shortFileName.
   */
  public static File getResourceFile(final String shortFileName) {
    Objects.requireNonNull(shortFileName, "input parameter 'String shortFileName' cannot be null.");
    final String slashName = (shortFileName.charAt(0) == '/') ? shortFileName : '/' + shortFileName;
    final URL url = TestUtil.class.getResource(slashName);
    Objects.requireNonNull(url, "resource " + slashName + " returns null URL.");
    File file;
    file = createTempFile(slashName);
    if (url.getProtocol().equals("jar")) { //definitely a jar
      try (final InputStream input = TestUtil.class.getResourceAsStream(slashName);
        final OutputStream out = new FileOutputStream(file)) {
        Objects.requireNonNull(input, "InputStream  is null.");
        int numRead = 0;
        final byte[] buf = new byte[1024];
        while ((numRead = input.read(buf)) != -1) { out.write(buf, 0, numRead); }
      } catch (final IOException e ) { throw new RuntimeException(e); }
    } else { //protocol says resource is not a jar, must be a file
      file = new File(getResourcePath(url));
    }
    if (!file.setReadable(false, true)) {
      throw new IllegalStateException("Failed to set owner only 'Readable' on file");
    }
    if (!file.setWritable(false, false)) {
      throw new IllegalStateException("Failed to set everyone 'Not Writable' on file");
    }
    return file;
  }

  /**
   * Returns a byte array of the contents of the file defined by the given resource file's shortFileName.
   * @param shortFileName the last name in the pathname's name sequence.
   * @return a byte array of the contents of the file defined by the given resource file's shortFileName.
   * @throws IllegalArgumentException if resource cannot be read.
   */
  public static byte[] getResourceBytes(final String shortFileName) {
    Objects.requireNonNull(shortFileName, "input parameter 'String shortFileName' cannot be null.");
    final String slashName = (shortFileName.charAt(0) == '/') ? shortFileName : '/' + shortFileName;
    final URL url = TestUtil.class.getResource(slashName);
    Objects.requireNonNull(url, "resource " + slashName + " returns null URL.");
    final byte[] out;
    if (url.getProtocol().equals("jar")) { //definitely a jar
      try (final InputStream input = TestUtil.class.getResourceAsStream(slashName)) {
        out = readAllBytesFromInputStream(input);
      } catch (final IOException e) { throw new RuntimeException(e); }
    } else { //protocol says resource is not a jar, must be a file
      try {
        out = Files.readAllBytes(Paths.get(getResourcePath(url)));
      } catch (final IOException e) { throw new RuntimeException(e); }
    }
    return out;
  }

  /**
   * Note: This is only needed in Java 8 as it is part of Java 9+.
   * Read all bytes from the given <i>InputStream</i>.
   * This is limited to streams that are no longer than the maximum allocatable byte array determined by the VM.
   * This may be a little smaller than <i>Integer.MAX_VALUE</i>.
   * @param in the Input Stream
   * @return byte array
   */
  public static byte[] readAllBytesFromInputStream(final InputStream in) {
    return readBytesFromInputStream(Integer.MAX_VALUE, in);
  }

  /**
   * Note: This is only needed in Java 8 as is part of Java 9+.
   * Read <i>numBytesToRead</i> bytes from an input stream into a single byte array.
   * This is limited to streams that are no longer than the maximum allocatable byte array determined by the VM.
   * This may be a little smaller than <i>Integer.MAX_VALUE</i>.
   * @param numBytesToRead number of bytes to read
   * @param in the InputStream
   * @return the filled byte array from the input stream
   * @throws IllegalArgumentException if array size grows larger than what can be safely allocated by some VMs.

   */
  public static byte[] readBytesFromInputStream(final int numBytesToRead, final InputStream in) {
    if (numBytesToRead < 0) { throw new IllegalArgumentException("numBytesToRead must be positive or zero."); }

    List<byte[]> buffers = null;
    byte[] result = null;
    int totalBytesRead = 0;
    int remaining = numBytesToRead;
    int chunkCnt;
    do {
        final byte[] partialBuffer = new byte[Math.min(remaining, BUF_SIZE)];
        int numRead = 0;

        try {
          // reads input stream in chunks of partial buffers, stops at EOF or when remaining is zero.
          while ((chunkCnt =
                in.read(partialBuffer, numRead, Math.min(partialBuffer.length - numRead, remaining))) > 0) {
              numRead += chunkCnt;
              remaining -= chunkCnt;
          }
        } catch (final IOException e) { throw new RuntimeException(e); }

        if (numRead > 0) {
            if (Integer.MAX_VALUE - Long.BYTES - totalBytesRead < numRead) {
              throw new IllegalArgumentException(
                  "Input stream is larger than what can be safely allocated as a byte[] in some VMs."); }
            totalBytesRead += numRead;
            if (result == null) {
                result = partialBuffer;
            } else {
                if (buffers == null) {
                    buffers = new ArrayList<>();
                    buffers.add(result);
                }
                buffers.add(partialBuffer);
            }
        }
    } while (chunkCnt >= 0 && remaining > 0);

    final byte[] out;
    if (buffers == null) {
        if (result == null) {
          out = new byte[0];
        } else {
          out = result.length == totalBytesRead ? result : Arrays.copyOf(result, totalBytesRead);
        }
        return out;
    }

    result = new byte[totalBytesRead];
    int offset = 0;
    remaining = totalBytesRead;
    for (byte[] b : buffers) {
        final int count = Math.min(b.length, remaining);
        System.arraycopy(b, 0, result, offset, count);
        offset += count;
        remaining -= count;
    }
    return result;
  }

  private static String getResourcePath(final URL url) { //must not be null
    try {
      final URI uri = url.toURI();
      //decodes any special characters
      final String path = uri.isAbsolute() ? Paths.get(uri).toAbsolutePath().toString() : uri.getPath();
      return path;
    } catch (final URISyntaxException e) {
      throw new IllegalArgumentException("Cannot find resource: " + url.toString() + Util.LS + e);
    }
  }

  /**
   * Create an empty temporary file.
   * On a Mac these files are stored at the system variable $TMPDIR.  They should be cleared on a reboot.
   * @param shortFileName the name before prefixes and suffixes are added here and by the OS.
   * The final extension will be the current extension. The prefix "temp_" is added here.
   * @return a temp file,which will be eventually deleted by the OS
   */
  private static File createTempFile(final String shortFileName) {
    //remove any leading slash
    final String resName = (shortFileName.charAt(0) == '/') ? shortFileName.substring(1) : shortFileName;
    final String suffix;
    final String name;
    final int  lastIdx = resName.length() - 1;
    final int lastIdxOfDot = resName.lastIndexOf('.');
    if (lastIdxOfDot == -1) {
      suffix = ".tmp";
      name = resName;
    } else if (lastIdxOfDot == lastIdx) {
      suffix = ".tmp";
      name = resName.substring(0, lastIdxOfDot);
    } else { //has a real suffix
      suffix = resName.substring(lastIdxOfDot);
      name = resName.substring(0, lastIdxOfDot);
    }
    final File file;
    try {
      file = File.createTempFile("temp_" + name, suffix);
      if (!file.setReadable(false, true)) {
        throw new IllegalStateException("Failed to set only owner 'Readable' on file");
      }
      if (!file.setWritable(false, true)) {
        throw new IllegalStateException("Failed to set only owner 'Writable' on file");
      }

    } catch (final IOException e) { throw new RuntimeException(e); }
    return file;
  }

}