@inproceedings{8e842da95614427ab0150639b9a8e109,
title = "Massively scalable near duplicate detection in streams of documents using MDSH",
abstract = "In a world where large-scale text collections are not only becoming ubiquitous but also are growing at increasing rates, near duplicate documents are becoming a growing concern that has the potential to hinder many different information filtering tasks. While others have tried to address this problem, prior techniques have only been used on limited collection sizes and static cases. We will briefly describe the problem in the context of Open Source analysis along with our additional constraints for performance. In this work we propose two variations on Multi-dimensional Spectral Hash (MDSH) tailored for working on extremely large, growing sets of text documents. We analyze the memory and runtime characteristics of our techniques and provide an informal analysis of the quality of the near-duplicate clusters produced by our techniques.",
keywords = "Big Data, MDSH, Near Duplicate Detection, Open Source Intelligence, Streaming Text",
author = "{Logasa Bogen}, Paul and Symons, {Christopher T.} and Amber McKenzie and Patton, {Robert M.} and Gillen, {Robert E.}",
year = "2013",
doi = "10.1109/BigData.2013.6691610",
language = "English",
isbn = "9781479912926",
series = "Proceedings - 2013 IEEE International Conference on Big Data, Big Data 2013",
publisher = "IEEE Computer Society",
pages = "480--486",
booktitle = "Proceedings - 2013 IEEE International Conference on Big Data, Big Data 2013",
note = "2013 IEEE International Conference on Big Data, Big Data 2013 ; Conference date: 06-10-2013 Through 09-10-2013",
}