@inproceedings{511b7354c18c4b55b6549b3121ae04d9,
title = "Learning concave-convex profiles of data transport over dedicated connections",
abstract = "Dedicated data transport infrastructures are increasingly being deployed to support distributed big-data and high-performance computing scenarios. These infrastructures employ data transfer nodes that use sophisticated software stacks to support network transport among sites, which often house distributed file and storage systems. Throughput measurements collected over such infrastructures for a range of round trip times (RTTs) reflect the underlying complex end-to-end connections, and have revealed dichotomous throughput profiles as functions of RTT. In particular, concave regions of throughput profiles at lower RTTs indicate near-optimal performance, and convex regions at higher RTTs indicate bottlenecks due to factors such as buffer or credit limits. We present a machine learning method that explicitly infers these concave and convex regions and transitions between them using sigmoid functions. We also provide distribution-free confidence estimates for the generalization error of these concave-convex profile estimates. Throughput profiles for data transfers over 10{\^A} Gbps connections with 0#x2013;366ms RTT provide important performance insights, including the near optimality of transfers performed with the XDD tool between XFS filesystems, and the performance limits of wide-area Lustre extensions using LNet routers. A direct application of generic machine learning packages does not adequately highlight these critical performance regions or provide as precise confidence estimates.",
keywords = "Concavity-convexity, Data transport, Generalization bounds, Throughput profile",
author = "Rao, {Nageswara S.V.} and Satyabrata Sen and Zhengchun Liu and Rajkumar Kettimuthu and Ian Foster",
note = "Publisher Copyright: {\textcopyright} Springer Nature Switzerland AG 2019.; 1st International Conference on Machine Learning for Networking, MLN 2018 ; Conference date: 27-11-2018 Through 29-11-2018",
year = "2019",
doi = "10.1007/978-3-030-19945-6_1",
language = "English",
isbn = "9783030199449",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Verlag",
pages = "1--22",
editor = "{\'E}ric Renault and Selma Boumerdassi and Paul M{\"u}hlethaler",
booktitle = "Machine Learning for Networking - 1st International Conference, MLN 2018, Revised Selected Papers",
}