@inproceedings{c465fd53d6fb432eb02776a7247a7671,
title = "Turbine: A distributed-memory dataflow engine for extreme-scale many-task applications",
abstract = "Efficiently utilizing the rapidly increasing concurrency of multi-petaop computing systems is a significant program- ming challenge. One approach is to structure applications with an upper layer of many loosely-coupled coarse-grained tasks, each comprising a tightly-coupled parallel function or program. {"}Many-task{"} programming models such as functional parallel dataflow may be used at the upper layer to generate massive numbers of tasks, each of which generates significant tighly-coupled parallelism at the lower level via multithreading, message passing, and/or partitioned global address spaces. At large scales, however, the management of task distribution, data dependencies, and inter-task data movement is a significant performance challenge. In this work, we describe Turbine, a new highly scalable and distributed many-task dataflow engine. Turbine executes a generalized many-task intermediate representation with automated self-distribution, and is scalable to multi-petaop infrastructures. We present here the architecture of Turbine and its performance on highly concurrent systems.",
keywords = "ADLB, Concurrency, Dataflow, Exascale, MPI, Swift, Turbine",
author = "Wozniak, {Justin M.} and Armstrong, {Timothy G.} and Ketan Maheshwari and Lusk, {Ewing L.} and Katz, {Daniel S.} and Michael Wilde and Foster, {Ian T.}",
year = "2012",
doi = "10.1145/2443416.2443421",
language = "English",
isbn = "9781450318761",
series = "ACM International Conference Proceeding Series",
booktitle = "Proceedings of the 1st ACM SIGMOD Workshop on Scalable Workflow Execution Engines and Technologies, SWEET 2012",
note = "1st ACM SIGMOD Workshop on Scalable Workflow Execution Engines and Technologies, SWEET 2012 ; Conference date: 20-05-2012 Through 20-05-2012",
}