Support importing time profiles from Instruments .trace files (#41)

#33 added support for importing from instruments indirectly via opening instruments and using the deep copy command. This PR adds support for importing `.trace` files directly, though only for time profiles specifically, and only for the highest sample count thread in the profile.

This PR adds `.trace` files from Instruments 9, and adds support for importing from either Instruments 8 and 9. The only major difference in the file format seems to be that Instruments 9 applies raw `zlib` compression generously throughout the file.

This PR also adds example `.trace` files for memory allocations, which are not supported for direct import. They use a totally different storage format for recording memory allocations, and I haven't yet figured out how that list of allocations references their corresponding callstack.

Lastly, this PR also adds examples from Instruments 7 since I happen to have a machine with an old version of Instruments. Import from Instruments 7 probably wouldn't be hard to add, but I haven't done that in this PR.

This currently only works in Chrome, and only via drag-and-drop of the files.

To test, drag the decompressed `simple-time-profile.trace` from 6016d970b9/sample/profiles/Instruments/9.3.1/simple-time-profile.trace.zip onto speedscope.

The result should be this:

![image](https://user-images.githubusercontent.com/150329/40162338-8fa13502-5968-11e8-8fb3-40626e41884a.png)

Fixes #15

application.tsx

@@ -2,9 +2,13 @@ import {h} from 'preact'
 import {StyleSheet, css} from 'aphrodite'
 import {ReloadableComponent, SerializedComponent} from './reloadable'
 
+// TODO(jlfwong): Load these async, since none of them are required for initial render
 import {importFromBGFlameGraph} from './import/bg-flamegraph'
 import {importFromStackprof} from './import/stackprof'
 import {importFromChromeTimeline, importFromChromeCPUProfile} from './import/chrome'
+import {importFromFirefox} from './import/firefox'
+import {importFromInstrumentsDeepCopy, importFromInstrumentsTrace} from './import/instruments'
+
 import {FlamechartRenderer} from './flamechart-renderer'
 import {CanvasContext} from './canvas-context'
 
@@ -13,8 +17,6 @@ import {Flamechart} from './flamechart'
 import {FlamechartView} from './flamechart-view'
 import {FontFamily, FontSize, Colors} from './style'
 import {getHashParams, HashParams} from './hash-params'
-import {importFromFirefox} from './import/firefox'
-import {importFromInstrumentsDeepCopy} from './import/instruments'
 
 declare function require(x: string): any
 const exampleProfileURL = require('./sample/profiles/stackcollapse/perf-vertx-stacks-01-collapsed-all.txt')
@@ -39,7 +41,7 @@ interface ToolbarProps extends ApplicationState {
   setSortOrder(order: SortOrder): void
 }
 
-function importProfile(contents: string, fileName: string): Profile | null {
+function importProfile(fileName: string, contents: string): Profile | null {
   try {
     // First pass: Check known file format names to infer the file type
     if (fileName.endsWith('.cpuprofile')) {
@@ -252,21 +254,35 @@ export class Application extends ReloadableComponent<{}, ApplicationState> {
     }
   }
 
-  async loadFromString(fileName: string, contents: string) {
+  async loadProfile(loader: () => Promise<Profile | null>) {
+    await new Promise(resolve => this.setState({loading: true}, resolve))
+    await new Promise(resolve => setTimeout(resolve, 0))
+
     if (!this.canvasContext) return
 
     console.time('import')
-    const profile = importProfile(contents, fileName)
+
+    let profile: Profile | null = null
+    try {
+      profile = await loader()
+    } catch (e) {
+      alert('Failed to load format. See console for details')
+      console.log(e)
+      this.setState({error: true})
+      return
+    }
+
     if (profile == null) {
-      this.setState({loading: false})
       // TODO(jlfwong): Make this a nicer overlay
       alert('Unrecognized format! See documentation about supported formats.')
       return
     }
 
+    console.log('PROFILE', profile)
+
     await profile.demangle()
 
-    const title = this.hashParams.title || fileName
+    const title = this.hashParams.title || profile.getName()
     profile.setName(title)
     document.title = `${title} - speedscope`
 
@@ -322,33 +338,48 @@ export class Application extends ReloadableComponent<{}, ApplicationState> {
   }
 
   loadFromFile(file: File) {
-    this.setState({loading: true}, () => {
-      requestAnimationFrame(() => {
-        const reader = new FileReader()
-        reader.addEventListener('loadend', () => {
-          this.loadFromString(file.name, reader.result)
-        })
-        reader.readAsText(file)
-      })
-    })
+    this.loadProfile(
+      () =>
+        new Promise((resolve, reject) => {
+          const reader = new FileReader()
+          reader.addEventListener('loadend', () => {
+            const profile = importProfile(file.name, reader.result)
+            if (profile) resolve(profile)
+            else reject()
+          })
+          reader.readAsText(file)
+        }),
+    )
   }
 
   loadExample = () => {
-    this.setState({loading: true})
-    const filename = 'perf-vertx-stacks-01-collapsed-all.txt'
-    fetch(exampleProfileURL)
-      .then(resp => resp.text())
-      .then(data => {
-        this.loadFromString(filename, data)
-      })
+    this.loadProfile(async () => {
+      const filename = 'perf-vertx-stacks-01-collapsed-all.txt'
+      return await fetch(exampleProfileURL)
+        .then(resp => resp.text())
+        .then(data => importProfile(filename, data))
+    })
   }
 
   onDrop = (ev: DragEvent) => {
+    ev.preventDefault()
+
+    const firstItem = ev.dataTransfer.items[0]
+    if ('webkitGetAsEntry' in firstItem) {
+      const webkitEntry: WebKitEntry = firstItem.webkitGetAsEntry()
+
+      // Instrument.app file format is actually a directory.
+      if (webkitEntry.isDirectory && webkitEntry.name.endsWith('.trace')) {
+        console.log('Importing as Instruments.app .trace file')
+        this.loadProfile(async () => await importFromInstrumentsTrace(webkitEntry))
+        return
+      }
+    }
+
     let file: File | null = ev.dataTransfer.files.item(0)
     if (file) {
       this.loadFromFile(file)
     }
-    ev.preventDefault()
   }
 
   onDragOver = (ev: DragEvent) => {
@@ -372,12 +403,7 @@ export class Application extends ReloadableComponent<{}, ApplicationState> {
     ev.stopPropagation()
 
     const pasted = (ev as ClipboardEvent).clipboardData.getData('text')
-    this.setState({loading: true}, () => {
-      // Delay to allow the loading bar to display
-      setTimeout(() => {
-        this.loadFromString('From Clipboard', pasted)
-      }, 0)
-    })
+    this.loadProfile(async () => importProfile('From Clipboard', pasted))
   }
 
   componentDidMount() {
@@ -387,19 +413,15 @@ export class Application extends ReloadableComponent<{}, ApplicationState> {
   }
 
   async maybeLoadHashParamProfile() {
-    try {
-      if (this.hashParams.profileURL) {
+    if (this.hashParams.profileURL) {
+      this.loadProfile(async () => {
         const response = await fetch(this.hashParams.profileURL)
-        const profile = await response.text()
-        let filename = new URL(this.hashParams.profileURL).pathname
+        let filename = new URL(this.hashParams.profileURL!).pathname
         if (filename.includes('/')) {
           filename = filename.slice(filename.lastIndexOf('/') + 1)
         }
-        await this.loadFromString(filename, profile)
-      }
-    } catch (e) {
-      this.setState({error: true})
-      throw e
+        return await importProfile(filename, await response.text())
+      })
     }
   }
 

import/instruments.ts

@@ -2,6 +2,8 @@
 // https://developer.apple.com/library/content/documentation/DeveloperTools/Conceptual/InstrumentsUserGuide/index.html
 
 import {Profile, FrameInfo, ByteFormatter, TimeFormatter} from '../profile'
+import {sortBy, getOrThrow, getOrInsert, lastOf, getOrElse, zeroPad} from '../utils'
+import * as pako from 'pako'
 
 function parseTSV<T>(contents: string): T[] {
   const lines = contents.split('\n').map(l => l.split('\t'))
@@ -98,7 +100,6 @@ export function importFromInstrumentsDeepCopy(contents: string): Profile {
     let stackDepth = symbolName.length - trimmedSymbolName.length
 
     if (stack.length - stackDepth < 0) {
-      console.log(stack, symbolName)
       throw new Error('Invalid format')
     }
 
@@ -139,3 +140,798 @@ export function importFromInstrumentsDeepCopy(contents: string): Profile {
 
   return profile
 }
+
+interface TraceDirectoryTree {
+  name: string
+  files: Map<string, File>
+  subdirectories: Map<string, TraceDirectoryTree>
+}
+
+async function extractDirectoryTree(entry: WebKitEntry): Promise<TraceDirectoryTree> {
+  const node: TraceDirectoryTree = {
+    name: entry.name,
+    files: new Map(),
+    subdirectories: new Map(),
+  }
+
+  const children = await new Promise<WebKitEntry[]>((resolve, reject) => {
+    ;(entry as any).createReader().readEntries((entries: any[]) => {
+      resolve(entries)
+    })
+  })
+
+  for (let child of children) {
+    if (child.isDirectory) {
+      const subtree = await extractDirectoryTree(child)
+      node.subdirectories.set(subtree.name, subtree)
+    } else {
+      const file = await new Promise<File>(resolve => (child as any).file(resolve))
+      node.files.set(file.name, file)
+    }
+  }
+
+  return node
+}
+
+class MaybeCompressedFileReader {
+  private fileData: Promise<ArrayBuffer>
+
+  constructor(file: File) {
+    this.fileData = new Promise(resolve => {
+      const reader = new FileReader()
+      reader.addEventListener('loadend', () => {
+        resolve(reader.result)
+      })
+      reader.readAsArrayBuffer(file)
+    })
+  }
+
+  private async getUncompressed(): Promise<ArrayBuffer> {
+    const fileData = await this.fileData
+    try {
+      const result = pako.inflate(new Uint8Array(fileData)).buffer
+      return result
+    } catch (e) {
+      return fileData
+    }
+  }
+
+  async readAsArrayBuffer(): Promise<ArrayBuffer> {
+    return await this.getUncompressed()
+  }
+
+  async readAsText(): Promise<string> {
+    const buffer = await this.getUncompressed()
+    let ret: string = ''
+
+    // JavaScript strings are UTF-16 encoded, but this data is coming
+    // from a UTF-8 encoded file.
+    const array = new Uint8Array(buffer)
+    for (let i = 0; i < array.length; i++) {
+      ret += String.fromCharCode(array[i])
+    }
+    return ret
+  }
+}
+
+function readAsArrayBuffer(file: File): Promise<ArrayBuffer> {
+  return new MaybeCompressedFileReader(file).readAsArrayBuffer()
+}
+
+function readAsText(file: File): Promise<string> {
+  return new MaybeCompressedFileReader(file).readAsText()
+}
+
+function getCoreDirForRun(tree: TraceDirectoryTree, selectedRun: number): TraceDirectoryTree {
+  const corespace = getOrThrow(tree.subdirectories, 'corespace')
+  const corespaceRunDir = getOrThrow(corespace.subdirectories, `run${selectedRun}`)
+  return getOrThrow(corespaceRunDir.subdirectories, 'core')
+}
+
+class BinReader {
+  private bytePos: number = 0
+  private view: DataView
+  constructor(buffer: ArrayBuffer) {
+    this.view = new DataView(buffer)
+  }
+  seek(pos: number) {
+    this.bytePos = pos
+  }
+  skip(byteCount: number) {
+    this.bytePos += byteCount
+  }
+  hasMore() {
+    return this.bytePos < this.view.byteLength
+  }
+  readUint8() {
+    this.bytePos++
+    if (this.bytePos > this.view.byteLength) return 0
+    return this.view.getUint8(this.bytePos - 1)
+  }
+
+  // Note: we intentionally use Math.pow here rather than bit shifts
+  // because JavaScript doesn't have true 64 bit integers.
+  readUint32() {
+    this.bytePos += 4
+    if (this.bytePos > this.view.byteLength) return 0
+    return this.view.getUint32(this.bytePos - 4, true)
+  }
+  readUint48() {
+    this.bytePos += 6
+    if (this.bytePos > this.view.byteLength) return 0
+
+    return (
+      this.view.getUint32(this.bytePos - 6, true) +
+      this.view.getUint16(this.bytePos - 2, true) * Math.pow(2, 32)
+    )
+  }
+  readUint64() {
+    this.bytePos += 8
+    if (this.bytePos > this.view.byteLength) return 0
+    return (
+      this.view.getUint32(this.bytePos - 8, true) +
+      this.view.getUint32(this.bytePos - 4, true) * Math.pow(2, 32)
+    )
+  }
+}
+
+interface Sample {
+  timestamp: number
+  threadID: number
+  backtraceID: number
+}
+
+async function getRawSampleList(core: TraceDirectoryTree): Promise<Sample[]> {
+  const stores = getOrThrow(core.subdirectories, 'stores')
+  for (let storedir of stores.subdirectories.values()) {
+    const schemaFile = storedir.files.get('schema.xml')
+    if (!schemaFile) continue
+    const schema = await readAsText(schemaFile)
+    if (!/name="time-profile"/.exec(schema)) {
+      continue
+    }
+    const bulkstore = new BinReader(
+      await readAsArrayBuffer(getOrThrow(storedir.files, 'bulkstore')),
+    )
+    // Ignore the first 3 words
+    bulkstore.readUint32()
+    bulkstore.readUint32()
+    bulkstore.readUint32()
+    const headerSize = bulkstore.readUint32()
+    const bytesPerEntry = bulkstore.readUint32()
+
+    bulkstore.seek(headerSize)
+
+    const samples: Sample[] = []
+    while (true) {
+      // Schema as of Instruments 8.3.3 is a 6 byte timestamp, followed by a bunch
+      // of stuff we don't care about, followed by a 4 byte backtrace ID
+      const timestamp = bulkstore.readUint48()
+      if (timestamp === 0) break
+
+      const threadID = bulkstore.readUint32()
+
+      bulkstore.skip(bytesPerEntry - 6 - 4 - 4)
+      const backtraceID = bulkstore.readUint32()
+      samples.push({timestamp, threadID, backtraceID})
+    }
+    return samples
+  }
+  throw new Error('Could not find sample list')
+}
+
+async function getIntegerArrays(samples: Sample[], core: TraceDirectoryTree): Promise<number[][]> {
+  const uniquing = getOrThrow(core.subdirectories, 'uniquing')
+  const arrayUniquer = getOrThrow(uniquing.subdirectories, 'arrayUniquer')
+  const integeruniquer = getOrThrow(arrayUniquer.files, 'integeruniquer.data')
+  const reader = new BinReader(await readAsArrayBuffer(integeruniquer))
+
+  let arrays: number[][] = []
+
+  // integeruniquer.data is a binary file containing an array of arrays of 64 bit integer.
+  // The schema is a 32 byte header followed by a stream of arrays.
+  // Each array consists of a 4 byte size N followed by N 8 byte little endian integers
+
+  // This table contains the memory addresses of stack frames
+
+  // Header we don't care about
+  reader.seek(32)
+
+  while (reader.hasMore()) {
+    let length = reader.readUint32()
+    let array: number[] = []
+    while (length--) {
+      array.push(reader.readUint64())
+    }
+    arrays.push(array)
+  }
+
+  return arrays
+}
+
+interface SymbolInfo {
+  symbolName: string | null
+  sourcePath: string | null
+  addressToLine: Map<number, number>
+}
+
+interface FormTemplateData {
+  selectedRun: number
+  instrument: string
+  version: number
+  addressToFrameMap: Map<number, FrameInfo>
+}
+
+async function readFormTemplate(tree: TraceDirectoryTree): Promise<FormTemplateData> {
+  const formTemplate = getOrThrow(tree.files, 'form.template')
+  const archive = readInstrumentsKeyedArchive(await readAsArrayBuffer(formTemplate))
+
+  const version = archive['com.apple.xray.owner.template.version']
+  const selectedRun = archive['com.apple.xray.owner.template'].get('_selectedRunNumber')
+  let instrument = archive['$1']
+  if ('stubInfoByUUID' in archive) {
+    instrument = Array.from(archive['stubInfoByUUID'].keys())[0]
+  }
+  let allRunData = archive['com.apple.xray.run.data']
+  const runData = getOrThrow<number, Map<any, any>>(
+    allRunData.runData,
+    allRunData.runNumbers.pop()!,
+  )
+
+  const symbolsByPid = getOrThrow<string, Map<number, {symbols: SymbolInfo[]}>>(
+    runData,
+    'symbolsByPid',
+  )
+
+  const addressToFrameMap = new Map<number, FrameInfo>()
+
+  // TODO(jlfwong): Deal with profiles with conflicts addresses?
+  for (let [_pid, symbols] of symbolsByPid.entries()) {
+    for (let symbol of symbols.symbols) {
+      if (!symbol) continue
+      const {sourcePath, symbolName, addressToLine} = symbol
+      for (let [address, _line] of addressToLine) {
+        getOrInsert(addressToFrameMap, address, () => {
+          const name = symbolName || `0x${zeroPad(address.toString(16), 16)}`
+          const frame: FrameInfo = {
+            key: `${sourcePath}:${name}`,
+            name: name,
+          }
+          if (sourcePath) {
+            frame.file = sourcePath
+          }
+          return frame
+        })
+      }
+    }
+  }
+
+  return {
+    version,
+    instrument,
+    selectedRun,
+    addressToFrameMap,
+  }
+}
+
+// Import from a .trace file saved from Mac Instruments.app
+export async function importFromInstrumentsTrace(entry: WebKitEntry): Promise<Profile> {
+  const tree = await extractDirectoryTree(entry)
+
+  const {version, selectedRun, instrument, addressToFrameMap} = await readFormTemplate(tree)
+  if (instrument !== 'com.apple.xray.instrument-type.coresampler2') {
+    throw new Error(
+      `The only supported instrument from .trace import is "com.apple.xray.instrument-type.coresampler2". Got ${instrument}`,
+    )
+  }
+  console.log('version: ', version)
+  console.log(`Importing time profile from run ${selectedRun}`)
+
+  const core = getCoreDirForRun(tree, selectedRun)
+  let samples = await getRawSampleList(core)
+  const arrays = await getIntegerArrays(samples, core)
+
+  const backtraceIDtoStack = new Map<number, FrameInfo[]>()
+
+  const profile = new Profile(lastOf(samples)!.timestamp)
+  profile.setName(entry.name)
+
+  // For now, we can only display the flamechart for a single thread of execution,
+  // So let's choose whichever thread had the most sample hits.
+  //
+  // TODO(jlfwong): Support displaying flamecharts for multiple threads.
+  const sampleCountByThreadID = new Map<number, number>()
+  for (let sample of samples) {
+    sampleCountByThreadID.set(
+      sample.threadID,
+      getOrElse(sampleCountByThreadID, sample.threadID, () => 0) + 1,
+    )
+  }
+  const counts = Array.from(sampleCountByThreadID.entries())
+  sortBy(counts, c => c[1])
+  const mainThreadID = lastOf(counts)![0]
+  samples = samples.filter(s => s.threadID === mainThreadID)
+
+  function appendRecursive(k: number, stack: FrameInfo[]) {
+    const frame = addressToFrameMap.get(k)
+    if (frame) {
+      stack.push(frame)
+    } else if (k in arrays) {
+      for (let addr of arrays[k]) {
+        appendRecursive(addr, stack)
+      }
+    } else {
+      const rawAddressFrame: FrameInfo = {
+        key: k,
+        name: `0x${zeroPad(k.toString(16), 16)}`,
+      }
+      addressToFrameMap.set(k, rawAddressFrame)
+      stack.push(rawAddressFrame)
+    }
+  }
+
+  let lastTimestamp: null | number = null
+  for (let sample of samples) {
+    const stackForSample = getOrInsert(backtraceIDtoStack, sample.backtraceID, id => {
+      const stack: FrameInfo[] = []
+      appendRecursive(id, stack)
+      stack.reverse()
+      return stack
+    })
+
+    if (lastTimestamp === null) {
+      // The first sample is sometimes fairly late in the profile for some reason.
+      // We'll just say nothing was known to be on the stack in that time.
+      profile.appendSample([], sample.timestamp)
+      lastTimestamp = sample.timestamp
+    }
+
+    if (sample.timestamp < lastTimestamp) {
+      throw new Error('Timestamps out of order!')
+    }
+
+    profile.appendSample(stackForSample, sample.timestamp - lastTimestamp)
+    lastTimestamp = sample.timestamp
+  }
+
+  profile.setValueFormatter(new TimeFormatter('nanoseconds'))
+  return profile
+}
+
+export function readInstrumentsKeyedArchive(buffer: ArrayBuffer): any {
+  const byteArray = new Uint8Array(buffer)
+  const parsedPlist = parseBinaryPlist(byteArray)
+  const data = expandKeyedArchive(parsedPlist, ($classname, object) => {
+    switch ($classname) {
+      case 'NSTextStorage':
+      case 'NSParagraphStyle':
+      case 'NSFont':
+        // Stuff that's irrelevant for constructing a flamegraph
+        return null
+
+      case 'PFTSymbolData': {
+        const ret = Object.create(null)
+        ret.symbolName = object.$0
+        ret.sourcePath = object.$1
+        ret.addressToLine = new Map<any, any>()
+        for (let i = 3; ; i += 2) {
+          const address = object['$' + i]
+          const line = object['$' + (i + 1)]
+          if (address == null || line == null) {
+            break
+          }
+          ret.addressToLine.set(address, line)
+        }
+        return ret
+      }
+
+      case 'PFTOwnerData': {
+        const ret = Object.create(null)
+        ret.ownerName = object.$0
+        ret.ownerPath = object.$1
+        return ret
+      }
+
+      case 'PFTPersistentSymbols': {
+        const ret = Object.create(null)
+        const symbolCount = object.$4
+
+        ret.threadNames = object.$3
+        ret.symbols = []
+        for (let i = 1; i < symbolCount; i++) {
+          ret.symbols.push(object['$' + (4 + i)])
+        }
+        return ret
+      }
+
+      case 'XRRunListData': {
+        const ret = Object.create(null)
+        ret.runNumbers = object.$0
+        ret.runData = object.$1
+        return ret
+      }
+
+      case 'XRIntKeyedDictionary': {
+        const ret = new Map()
+        const size = object.$0
+        for (let i = 0; i < size; i++) {
+          const key = object['$' + (1 + 2 * i)]
+          const value = object['$' + (1 + (2 * i + 1))]
+          ret.set(key, value)
+        }
+        return ret
+      }
+
+      case 'XRCore': {
+        const ret = Object.create(null)
+        ret.number = object.$0
+        ret.name = object.$1
+        return ret
+      }
+    }
+    return object
+  })
+  return data
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+export function decodeUTF8(bytes: Uint8Array): string {
+  let text = String.fromCharCode.apply(String, bytes)
+  if (text.slice(-1) === '\0') text = text.slice(0, -1) // Remove a single trailing null character if present
+  return decodeURIComponent(escape(text))
+}
+
+function isArray(value: any): boolean {
+  return value instanceof Array
+}
+
+function isDictionary(value: any): boolean {
+  return value !== null && typeof value === 'object' && Object.getPrototypeOf(value) === null
+}
+
+function followUID(objects: any[], value: any): any {
+  return value instanceof UID ? objects[value.index] : value
+}
+
+function expandKeyedArchive(
+  root: any,
+  interpretClass: ($classname: string, obj: any) => any = x => x,
+): any {
+  // Sanity checks
+  if (
+    root.$version !== 100000 ||
+    (root.$archiver !== 'MSArchiver' && root.$archiver !== 'NSKeyedArchiver') ||
+    !isDictionary(root.$top) ||
+    !isArray(root.$objects)
+  ) {
+    throw new Error('Invalid keyed archive')
+  }
+
+  // Substitute NSNull
+  if (root.$objects[0] === '$null') {
+    root.$objects[0] = null
+  }
+
+  // Pattern-match Objective-C constructs
+  for (let i = 0; i < root.$objects.length; i++) {
+    root.$objects[i] = paternMatchObjectiveC(root.$objects, root.$objects[i], interpretClass)
+  }
+
+  // Reconstruct the DAG from the parse tree
+  let visit = (object: any) => {
+    if (object instanceof UID) {
+      return root.$objects[object.index]
+    } else if (isArray(object)) {
+      for (let i = 0; i < object.length; i++) {
+        object[i] = visit(object[i])
+      }
+    } else if (isDictionary(object)) {
+      for (let key in object) {
+        object[key] = visit(object[key])
+      }
+    } else if (object instanceof Map) {
+      const clone = new Map(object)
+      object.clear()
+      for (let [k, v] of clone.entries()) {
+        object.set(visit(k), visit(v))
+      }
+    }
+    return object
+  }
+  for (let i = 0; i < root.$objects.length; i++) {
+    visit(root.$objects[i])
+  }
+  return visit(root.$top)
+}
+
+function paternMatchObjectiveC(
+  objects: any[],
+  value: any,
+  interpretClass: ($classname: string, obj: any) => any = x => x,
+): any {
+  if (isDictionary(value) && value.$class) {
+    let name = followUID(objects, value.$class).$classname
+    switch (name) {
+      case 'NSDecimalNumberPlaceholder': {
+        let length: number = value['NS.length']
+        let exponent: number = value['NS.exponent']
+        let byteOrder: number = value['NS.mantissa.bo']
+        let negative: boolean = value['NS.negative']
+        let mantissa = new Uint16Array(new Uint8Array(value['NS.mantissa']).buffer)
+        let decimal = 0
+
+        for (let i = 0; i < length; i++) {
+          let digit = mantissa[i]
+
+          if (byteOrder !== 1) {
+            // I assume this is how this works but I am unable to test it
+            digit = ((digit & 0xff00) >> 8) | ((digit & 0x00ff) << 8)
+          }
+
+          decimal += digit * Math.pow(65536, i)
+        }
+
+        decimal *= Math.pow(10, exponent)
+        return negative ? -decimal : decimal
+      }
+
+      // Replace NSData with a Uint8Array
+      case 'NSData':
+      case 'NSMutableData':
+        return value['NS.bytes'] || value['NS.data']
+
+      // Replace NSString with a string
+      case 'NSString':
+      case 'NSMutableString':
+        return decodeUTF8(value['NS.bytes'])
+
+      // Replace NSArray with an Array
+      case 'NSArray':
+      case 'NSMutableArray':
+        if ('NS.objects' in value) {
+          return value['NS.objects']
+        }
+        let array: any[] = []
+        while (true) {
+          let object = 'NS.object.' + array.length
+          if (!(object in value)) {
+            break
+          }
+          array.push(value[object])
+        }
+        return array
+
+      case '_NSKeyedCoderOldStyleArray': {
+        const count = value['NS.count']
+
+        // const size = value['NS.size']
+        // Types are encoded as single printable characters.
+        // See: https://github.com/apple/swift-corelibs-foundation/blob/76995e8d3d8c10f3f3ec344dace43426ab941d0e/Foundation/NSObjCRuntime.swift#L19
+        // const type = String.fromCharCode(value['NS.type'])
+
+        let array: any[] = []
+        for (let i = 0; i < count; i++) {
+          const element = value['$' + i]
+          array.push(element)
+        }
+        return array
+      }
+
+      case 'NSDictionary':
+      case 'NSMutableDictionary':
+        let map = new Map()
+        if ('NS.keys' in value && 'NS.objects' in value) {
+          for (let i = 0; i < value['NS.keys'].length; i++) {
+            map.set(value['NS.keys'][i], value['NS.objects'][i])
+          }
+        } else {
+          while (true) {
+            let key = 'NS.key.' + map.size
+            let object = 'NS.object.' + map.size
+            if (!(key in value) || !(object in value)) {
+              break
+            }
+            map.set(value[key], value[object])
+          }
+        }
+        return map
+
+      default:
+        const converted = interpretClass(name, value)
+        if (converted !== value) return converted
+    }
+  }
+  return value
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+export class UID {
+  constructor(public index: number) {}
+}
+
+function parseBinaryPlist(bytes: Uint8Array): any {
+  let text = 'bplist00'
+  for (let i = 0; i < 8; i++) {
+    if (bytes[i] !== text.charCodeAt(i)) {
+      throw new Error('File is not a binary plist')
+    }
+  }
+  return new BinaryPlistParser(
+    new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength),
+  ).parseRoot()
+}
+
+interface LengthAndOffset {
+  length: number
+  offset: number
+}
+
+// See http://opensource.apple.com/source/CF/CF-550/CFBinaryPList.c for details
+class BinaryPlistParser {
+  referenceSize = 0
+  objects: number[] = []
+  offsetTable: number[] = []
+
+  constructor(public view: DataView) {}
+
+  parseRoot(): any {
+    let trailer = this.view.byteLength - 32
+    let offsetSize = this.view.getUint8(trailer + 6)
+    this.referenceSize = this.view.getUint8(trailer + 7)
+
+    // Just use the last 32-bits of these 64-bit big-endian values
+    let objectCount = this.view.getUint32(trailer + 12, false)
+    let rootIndex = this.view.getUint32(trailer + 20, false)
+    let tableOffset = this.view.getUint32(trailer + 28, false)
+
+    // Parse all offsets before starting to parse objects
+    for (let i = 0; i < objectCount; i++) {
+      this.offsetTable.push(this.parseInteger(tableOffset, offsetSize))
+      tableOffset += offsetSize
+    }
+
+    // Parse the root object assuming the graph is a tree
+    return this.parseObject(this.offsetTable[rootIndex])
+  }
+
+  parseLengthAndOffset(offset: number, extra: number): LengthAndOffset {
+    if (extra !== 0x0f) return {length: extra, offset: 0}
+    let marker = this.view.getUint8(offset++)
+    if ((marker & 0xf0) !== 0x10)
+      throw new Error('Unexpected non-integer length at offset ' + offset)
+    let size = 1 << (marker & 0x0f)
+    return {length: this.parseInteger(offset, size), offset: size + 1}
+  }
+
+  parseSingleton(offset: number, extra: number): any {
+    if (extra === 0) return null
+    if (extra === 8) return false
+    if (extra === 9) return true
+    throw new Error('Unexpected extra value ' + extra + ' at offset ' + offset)
+  }
+
+  parseInteger(offset: number, size: number): number {
+    if (size === 1) return this.view.getUint8(offset)
+    if (size === 2) return this.view.getUint16(offset, false)
+    if (size === 4) return this.view.getUint32(offset, false)
+
+    if (size === 8) {
+      return (
+        Math.pow(2, 32 * 1) * this.view.getUint32(offset + 0, false) +
+        Math.pow(2, 32 * 0) * this.view.getUint32(offset + 4, false)
+      )
+    }
+
+    if (size === 16) {
+      return (
+        Math.pow(2, 32 * 3) * this.view.getUint32(offset + 0, false) +
+        Math.pow(2, 32 * 2) * this.view.getUint32(offset + 4, false) +
+        Math.pow(2, 32 * 1) * this.view.getUint32(offset + 8, false) +
+        Math.pow(2, 32 * 0) * this.view.getUint32(offset + 12, false)
+      )
+    }
+
+    throw new Error('Unexpected integer of size ' + size + ' at offset ' + offset)
+  }
+
+  parseFloat(offset: number, size: number): number {
+    if (size === 4) return this.view.getFloat32(offset, false)
+    if (size === 8) return this.view.getFloat64(offset, false)
+    throw new Error('Unexpected float of size ' + size + ' at offset ' + offset)
+  }
+
+  parseDate(offset: number, size: number): Date {
+    if (size !== 8) throw new Error('Unexpected date of size ' + size + ' at offset ' + offset)
+    let seconds = this.view.getFloat64(offset, false)
+    return new Date(978307200000 + seconds * 1000) // Starts from January 1st, 2001
+  }
+
+  parseData(offset: number, extra: number): Uint8Array {
+    let both = this.parseLengthAndOffset(offset, extra)
+    return new Uint8Array(this.view.buffer, offset + both.offset, both.length)
+  }
+
+  parseStringASCII(offset: number, extra: number): string {
+    let both = this.parseLengthAndOffset(offset, extra)
+    let text = ''
+    offset += both.offset
+    for (let i = 0; i < both.length; i++) {
+      text += String.fromCharCode(this.view.getUint8(offset++))
+    }
+    return text
+  }
+
+  parseStringUTF16(offset: number, extra: number): string {
+    let both = this.parseLengthAndOffset(offset, extra)
+    let text = ''
+    offset += both.offset
+    for (let i = 0; i < both.length; i++) {
+      text += String.fromCharCode(this.view.getUint16(offset, false))
+      offset += 2
+    }
+    return text
+  }
+
+  parseUID(offset: number, size: number): UID {
+    return new UID(this.parseInteger(offset, size))
+  }
+
+  parseArray(offset: number, extra: number): any[] {
+    let both = this.parseLengthAndOffset(offset, extra)
+    let array: any[] = []
+    let size = this.referenceSize
+    offset += both.offset
+    for (let i = 0; i < both.length; i++) {
+      array.push(this.parseObject(this.offsetTable[this.parseInteger(offset, size)]))
+      offset += size
+    }
+    return array
+  }
+
+  parseDictionary(offset: number, extra: number): Object {
+    let both = this.parseLengthAndOffset(offset, extra)
+    let dictionary = Object.create(null)
+    let size = this.referenceSize
+    let nextKey = offset + both.offset
+    let nextValue = nextKey + both.length * size
+    for (let i = 0; i < both.length; i++) {
+      let key = this.parseObject(this.offsetTable[this.parseInteger(nextKey, size)])
+      let value = this.parseObject(this.offsetTable[this.parseInteger(nextValue, size)])
+      if (typeof key !== 'string') throw new Error('Unexpected non-string key at offset ' + nextKey)
+      dictionary[key] = value
+      nextKey += size
+      nextValue += size
+    }
+    return dictionary
+  }
+
+  parseObject(offset: number): any {
+    let marker = this.view.getUint8(offset++)
+    let extra = marker & 0x0f
+    switch (marker >> 4) {
+      case 0x0:
+        return this.parseSingleton(offset, extra)
+      case 0x1:
+        return this.parseInteger(offset, 1 << extra)
+      case 0x2:
+        return this.parseFloat(offset, 1 << extra)
+      case 0x3:
+        return this.parseDate(offset, 1 << extra)
+      case 0x4:
+        return this.parseData(offset, extra)
+      case 0x5:
+        return this.parseStringASCII(offset, extra)
+      case 0x6:
+        return this.parseStringUTF16(offset, extra)
+      case 0x8:
+        return this.parseUID(offset, extra + 1)
+      case 0xa:
+        return this.parseArray(offset, extra)
+      case 0xd:
+        return this.parseDictionary(offset, extra)
+    }
+    throw new Error('Unexpected marker ' + marker + ' at offset ' + --offset)
+  }
+}

package-lock.json

@@ -4,6 +4,12 @@
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
+    "@types/pako": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/pako/-/pako-1.0.0.tgz",
+      "integrity": "sha1-6q6DZNG391LiY7w/1o3+yY5hNsU=",
+      "dev": true
+    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -5249,9 +5255,9 @@
       "dev": true
     },
     "pako": {
-      "version": "0.2.9",
-      "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
-      "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=",
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz",
+      "integrity": "sha512-lQe48YPsMJAig+yngZ87Lus+NF+3mtu7DVOBu6b/gHO1YpKwIj5AWjZ/TOS7i46HD/UixzWb1zeWDZfGZ3iYcg==",
       "dev": true
     },
     "parcel-bundler": {
@@ -8893,6 +8899,14 @@
       "requires": {
         "pako": "0.2.9",
         "tiny-inflate": "1.0.2"
+      },
+      "dependencies": {
+        "pako": {
+          "version": "0.2.9",
+          "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
+          "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=",
+          "dev": true
+        }
       }
     },
     "union-value": {

package.json

@@ -26,6 +26,8 @@
     "regl": "1.3.1",
     "typescript": "2.8.1",
     "typescript-eslint-parser": "^14.0.0",
-    "uglify-es": "3.2.2"
+    "uglify-es": "3.2.2",
+    "@types/pako": "1.0.0",
+    "pako": "1.0.6"
   }
 }

sample/programs/cpp/.gitignore

@@ -1,2 +1,4 @@
 simple
-simple.dSYM
+random
+*.dSYM
+

sample/programs/cpp/Makefile

@@ -1,2 +1,9 @@
+.PHONY: all
+
+all: simple random
+
 simple: simple.cpp
 	g++ -Wall -g -o $@ $<
+
+random: random.cpp
+	g++ -Wall -g -o $@ $<

sample/programs/cpp/random.cpp

@@ -0,0 +1,52 @@
+#include <cstdlib>
+using namespace std;
+
+void leakMemory() {
+  for (int i = 0, ii = rand() % 27; i < ii; i++) {
+    malloc(rand() % (10 * 1024));
+  }
+}
+
+void alpha() {
+  leakMemory();
+  int z = 3;
+  for (int i = 0, ii = rand() % 100000; i < ii; i++) {
+    z *= 3;
+  }
+}
+
+void beta() {
+  leakMemory();
+  int z = 3;
+  for (int i = 0, ii = rand() % 30000; i < ii; i++) {
+    z *= 3;
+  }
+}
+
+void delta() {
+  leakMemory();
+  int z = 3;
+  for (int i = 0, ii = rand() % 10000; i < ii; i++) {
+    z *= 3;
+  }
+  alpha();
+  beta();
+}
+
+void gamma() {
+  leakMemory();
+  int z = 3;
+  for (int i = 0, ii = rand() % 70000; i < ii; i++) {
+    z *= 3;
+  }
+}
+
+int main(int argc, char* argv[]) {
+  while (true) {
+    alpha();
+    beta();
+    delta();
+    gamma();
+  }
+  return 0;
+}

utils.ts

@@ -19,6 +19,13 @@ export function getOrElse<K, V>(map: Map<K, V>, k: K, fallback: (k?: K) => V): V
   return map.get(k)!
 }
 
+export function getOrThrow<K, V>(map: Map<K, V>, k: K): V {
+  if (!map.has(k)) {
+    throw new Error(`Expected key ${k}`)
+  }
+  return map.get(k)!
+}
+
 export function* itMap<T, U>(it: Iterable<T>, f: (t: T) => U): Iterable<U> {
   for (let t of it) {
     yield f(t)
@@ -39,6 +46,10 @@ export function itReduce<T, U>(it: Iterable<T>, f: (a: U, b: T) => U, init: U):
   return accum
 }
 
+export function zeroPad(s: string, width: number) {
+  return new Array(Math.max(width - s.length, 0) + 1).join('0') + s
+}
+
 // NOTE: This blindly assumes the same result across contexts.
 const measureTextCache = new Map<string, number>()