Subzero: Fix build problem against LLVM trunk.

pnacl-llvm/DataStream.cpp

+//===--- llvm/Support/DataStream.cpp - Lazy streamed data -----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements DataStreamer, which fetches bytes of Data from
+// a stream source. It provides support for streaming (lazy reading) of
+// bitcode. An example implementation of streaming from a file or stdin
+// is included.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/DataStream.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Program.h"
+#include <string>
+#include <system_error>
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+using namespace llvm;
+
+#define DEBUG_TYPE "Data-stream"
+
+// Interface goals:
+// * StreamingMemoryObject doesn't care about complexities like using
+//   threads/async callbacks to actually overlap download+compile
+// * Don't want to duplicate Data in memory
+// * Don't need to know total Data len in advance
+// Non-goals:
+// StreamingMemoryObject already has random access so this interface only does
+// in-order streaming (no arbitrary seeking, else we'd have to buffer all the
+// Data here in addition to MemoryObject).  This also means that if we want
+// to be able to to free Data, BitstreamBytes/BitcodeReader will implement it
+
+STATISTIC(NumStreamFetches, "Number of calls to Data stream fetch");
+
+namespace llvm {
+DataStreamer::~DataStreamer() {}
+}
+
+namespace {
+
+// Very simple stream backed by a file. Mostly useful for stdin and debugging;
+// actual file access is probably still best done with mmap.
+class DataFileStreamer : public DataStreamer {
+ int Fd;
+public:
+  DataFileStreamer() : Fd(0) {}
+  ~DataFileStreamer() override { close(Fd); }
+  size_t GetBytes(unsigned char *buf, size_t len) override {
+    NumStreamFetches++;
+    return read(Fd, buf, len);
+  }
+
+  std::error_code OpenFile(const std::string &Filename) {
+    if (Filename == "-") {
+      Fd = 0;
+      sys::ChangeStdinToBinary();
+      return std::error_code();
+    }
+
+    return sys::fs::openFileForRead(Filename, Fd);
+  }
+};
+
+}
+
+std::unique_ptr<DataStreamer>
+llvm::getDataFileStreamer(const std::string &Filename, std::string *StrError) {
+  std::unique_ptr<DataFileStreamer> s = make_unique<DataFileStreamer>();
+  if (std::error_code e = s->OpenFile(Filename)) {
+    *StrError = std::string("Could not open ") + Filename + ": " +
+        e.message() + "\n";
+    return nullptr;
+  }
+  return std::move(s);
+}

pnacl-llvm/MemoryObject.cpp

+//===- MemoryObject.cpp - Abstract memory interface -----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/MemoryObject.h"
+using namespace llvm;
+  
+MemoryObject::~MemoryObject() {
+}

pnacl-llvm/StreamingMemoryObject.cpp

+//===- StreamingMemoryObject.cpp - Streamable data interface -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/StreamingMemoryObject.h"
+#include <cassert>
+#include <cstddef>
+#include <cstring>
+using namespace llvm;
+
+namespace {
+
+class RawMemoryObject : public MemoryObject {
+public:
+  RawMemoryObject(const unsigned char *Start, const unsigned char *End) :
+    FirstChar(Start), LastChar(End) {
+    assert(LastChar >= FirstChar && "Invalid start/end range");
+  }
+
+  uint64_t getExtent() const override {
+    return LastChar - FirstChar;
+  }
+  uint64_t readBytes(uint8_t *Buf, uint64_t Size,
+                     uint64_t Address) const override;
+  const uint8_t *getPointer(uint64_t address, uint64_t size) const override;
+  bool isValidAddress(uint64_t address) const override {
+    return validAddress(address);
+  }
+
+private:
+  const uint8_t* const FirstChar;
+  const uint8_t* const LastChar;
+
+  // These are implemented as inline functions here to avoid multiple virtual
+  // calls per public function
+  bool validAddress(uint64_t address) const {
+    return static_cast<std::ptrdiff_t>(address) < LastChar - FirstChar;
+  }
+
+  RawMemoryObject(const RawMemoryObject&) = delete;
+  void operator=(const RawMemoryObject&) = delete;
+};
+
+uint64_t RawMemoryObject::readBytes(uint8_t *Buf, uint64_t Size,
+                                    uint64_t Address) const {
+  uint64_t BufferSize = LastChar - FirstChar;
+  if (Address >= BufferSize)
+    return 0;
+
+  uint64_t End = Address + Size;
+  if (End > BufferSize)
+    End = BufferSize;
+
+  assert(static_cast<int64_t>(End - Address) >= 0);
+  Size = End - Address;
+  memcpy(Buf, Address + FirstChar, Size);
+  return Size;
+}
+
+const uint8_t *RawMemoryObject::getPointer(uint64_t address,
+                                           uint64_t size) const {
+  return FirstChar + address;
+}
+} // anonymous namespace
+
+namespace llvm {
+// If the bitcode has a header, then its size is known, and we don't have to
+// block until we actually want to read it.
+bool StreamingMemoryObject::isValidAddress(uint64_t address) const {
+  if (ObjectSize && address < ObjectSize) return true;
+  return fetchToPos(address);
+}
+
+uint64_t StreamingMemoryObject::getExtent() const {
+  if (ObjectSize) return ObjectSize;
+  size_t pos = BytesRead + kChunkSize;
+  // keep fetching until we run out of bytes
+  while (fetchToPos(pos)) pos += kChunkSize;
+  return ObjectSize;
+}
+
+uint64_t StreamingMemoryObject::readBytes(uint8_t *Buf, uint64_t Size,
+                                          uint64_t Address) const {
+  fetchToPos(Address + Size - 1);
+  // Note: For wrapped bitcode files will set ObjectSize after the
+  // first call to fetchToPos. In such cases, ObjectSize can be
+  // smaller than BytesRead.
+  size_t MaxAddress =
+      (ObjectSize && ObjectSize < BytesRead) ? ObjectSize : BytesRead;
+  if (Address >= MaxAddress)
+    return 0;
+
+  uint64_t End = Address + Size;
+  if (End > MaxAddress)
+    End = MaxAddress;
+  assert(End >= Address);
+  Size = End - Address;
+  memcpy(Buf, &Bytes[Address + BytesSkipped], Size);
+  return Size;
+}
+
+const uint8_t *StreamingMemoryObject::getPointer(uint64_t Address,
+                                                 uint64_t Size) const {
+  fetchToPos(Address + Size - 1);
+  return &Bytes[Address + BytesSkipped];
+}
+
+bool StreamingMemoryObject::dropLeadingBytes(size_t s) {
+  if (BytesRead < s) return true;
+  BytesSkipped = s;
+  BytesRead -= s;
+  return false;
+}
+
+void StreamingMemoryObject::setKnownObjectSize(size_t size) {
+  ObjectSize = size;
+  Bytes.reserve(size);
+  if (ObjectSize <= BytesRead)
+    EOFReached = true;
+}
+
+MemoryObject *getNonStreamedMemoryObject(const unsigned char *Start,
+                                         const unsigned char *End) {
+  return new RawMemoryObject(Start, End);
+}
+
+StreamingMemoryObject::StreamingMemoryObject(
+    std::unique_ptr<DataStreamer> Streamer)
+    : Bytes(kChunkSize), Streamer(std::move(Streamer)), BytesRead(0),
+      BytesSkipped(0), ObjectSize(0), EOFReached(false) {
+  BytesRead = this->Streamer->GetBytes(&Bytes[0], kChunkSize);
+}
+}

pnacl-llvm/include/llvm/Support/DataStream.h

+//===---- llvm/Support/DataStream.h - Lazy bitcode streaming ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This header defines DataStreamer, which fetches bytes of data from
+// a stream source. It provides support for streaming (lazy reading) of
+// data, e.g. bitcode
+//
+//===----------------------------------------------------------------------===//
+
+
+#ifndef LLVM_SUPPORT_DATASTREAM_H
+#define LLVM_SUPPORT_DATASTREAM_H
+
+#include <memory>
+#include <string>
+
+namespace llvm {
+
+class DataStreamer {
+public:
+  /// Fetch bytes [start-end) from the stream, and write them to the
+  /// buffer pointed to by buf. Returns the number of bytes actually written.
+  virtual size_t GetBytes(unsigned char *buf, size_t len) = 0;
+
+  virtual ~DataStreamer();
+};
+
+std::unique_ptr<DataStreamer> getDataFileStreamer(const std::string &Filename,
+                                                  std::string *Err);
+}
+
+#endif  // LLVM_SUPPORT_DATASTREAM_H_

pnacl-llvm/include/llvm/Support/MemoryObject.h

+//===- MemoryObject.h - Abstract memory interface ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_MEMORYOBJECT_H
+#define LLVM_SUPPORT_MEMORYOBJECT_H
+
+#include "llvm/Support/DataTypes.h"
+
+namespace llvm {
+
+/// Interface to data which might be streamed. Streamability has 2 important
+/// implications/restrictions. First, the data might not yet exist in memory
+/// when the request is made. This just means that readByte/readBytes might have
+/// to block or do some work to get it. More significantly, the exact size of
+/// the object might not be known until it has all been fetched. This means that
+/// to return the right result, getExtent must also wait for all the data to
+/// arrive; therefore it should not be called on objects which are actually
+/// streamed (this would defeat the purpose of streaming). Instead,
+/// isValidAddress can be used to test addresses without knowing the exact size
+/// of the stream. Finally, getPointer can be used instead of readBytes to avoid
+/// extra copying.
+class MemoryObject {
+public:
+  virtual ~MemoryObject();
+
+  /// Returns the size of the region in bytes.  (The region is contiguous, so
+  /// the highest valid address of the region is getExtent() - 1).
+  ///
+  /// @result         - The size of the region.
+  virtual uint64_t getExtent() const = 0;
+
+  /// Tries to read a contiguous range of bytes from the region, up to the end
+  /// of the region.
+  ///
+  /// @param Buf      - A pointer to a buffer to be filled in.  Must be non-NULL
+  ///                   and large enough to hold size bytes.
+  /// @param Size     - The number of bytes to copy.
+  /// @param Address  - The address of the first byte, in the same space as
+  ///                   getBase().
+  /// @result         - The number of bytes read.
+  virtual uint64_t readBytes(uint8_t *Buf, uint64_t Size,
+                             uint64_t Address) const = 0;
+
+  /// Ensures that the requested data is in memory, and returns a pointer to it.
+  /// More efficient than using readBytes if the data is already in memory. May
+  /// block until (address - base + size) bytes have been read
+  /// @param address - address of the byte, in the same space as getBase()
+  /// @param size    - amount of data that must be available on return
+  /// @result        - valid pointer to the requested data
+  virtual const uint8_t *getPointer(uint64_t address, uint64_t size) const = 0;
+
+  /// Returns true if the address is within the object (i.e. between base and
+  /// base + extent - 1 inclusive). May block until (address - base) bytes have
+  /// been read
+  /// @param address - address of the byte, in the same space as getBase()
+  /// @result        - true if the address may be read with readByte()
+  virtual bool isValidAddress(uint64_t address) const = 0;
+};
+
+}
+
+#endif

pnacl-llvm/include/llvm/Support/StreamingMemoryObject.h

+//===- StreamingMemoryObject.h - Streamable data interface -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_STREAMINGMEMORYOBJECT_H
+#define LLVM_SUPPORT_STREAMINGMEMORYOBJECT_H
+
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/DataStream.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MemoryObject.h"
+#include <memory>
+#include <vector>
+
+namespace llvm {
+
+/// Interface to data which is actually streamed from a DataStreamer. In
+/// addition to inherited members, it has the dropLeadingBytes and
+/// setKnownObjectSize methods which are not applicable to non-streamed objects.
+class StreamingMemoryObject : public MemoryObject {
+public:
+  StreamingMemoryObject(std::unique_ptr<DataStreamer> Streamer);
+  uint64_t getExtent() const override;
+  uint64_t readBytes(uint8_t *Buf, uint64_t Size,
+                     uint64_t Address) const override;
+  const uint8_t *getPointer(uint64_t Address, uint64_t Size) const override;
+  bool isValidAddress(uint64_t address) const override;
+
+  /// Drop s bytes from the front of the stream, pushing the positions of the
+  /// remaining bytes down by s. This is used to skip past the bitcode header,
+  /// since we don't know a priori if it's present, and we can't put bytes
+  /// back into the stream once we've read them.
+  bool dropLeadingBytes(size_t s);
+
+  /// If the data object size is known in advance, many of the operations can
+  /// be made more efficient, so this method should be called before reading
+  /// starts (although it can be called anytime).
+  void setKnownObjectSize(size_t size);
+
+  /// The number of bytes read at a time from the data streamer.
+  static const uint32_t kChunkSize = 4096 * 4;
+
+private:
+  mutable std::vector<unsigned char> Bytes;
+  std::unique_ptr<DataStreamer> Streamer;
+  mutable size_t BytesRead;   // Bytes read from stream
+  size_t BytesSkipped;// Bytes skipped at start of stream (e.g. wrapper/header)
+  mutable size_t ObjectSize; // 0 if unknown, set if wrapper seen or EOF reached
+  mutable bool EOFReached;
+
+  // Fetch enough bytes such that Pos can be read (i.e. BytesRead >
+  // Pos). Returns true if Pos can be read.  Unlike most of the
+  // functions in BitcodeReader, returns true on success.  Most of the
+  // requests will be small, but we fetch at kChunkSize bytes at a
+  // time to avoid making too many potentially expensive GetBytes
+  // calls.
+  bool fetchToPos(size_t Pos) const {
+    while (Pos >= BytesRead) {
+      if (EOFReached)
+        return false;
+      Bytes.resize(BytesRead + BytesSkipped + kChunkSize);
+      size_t bytes = Streamer->GetBytes(&Bytes[BytesRead + BytesSkipped],
+                                        kChunkSize);
+      BytesRead += bytes;
+      if (bytes == 0) { // reached EOF/ran out of bytes
+        if (ObjectSize == 0)
+          ObjectSize = BytesRead;
+        EOFReached = true;
+      }
+    }
+    return !ObjectSize || Pos < ObjectSize;
+  }
+
+  StreamingMemoryObject(const StreamingMemoryObject&) = delete;
+  void operator=(const StreamingMemoryObject&) = delete;
+};
+
+MemoryObject *getNonStreamedMemoryObject(
+    const unsigned char *Start, const unsigned char *End);
+
+}
+#endif  // STREAMINGMEMORYOBJECT_H_