att_abstract={Crowd-sourcing is a promising method for fast and cheap transcription of large volumes of speech data. However, this method cannot achieve the accuracy of expert transcribers on speech that is difficult to transcribe. Faced with such speech data, we developed three new methods of crowd-sourcing allow explicit trade-offs among precision, recall, and cost. The methods are: incremental crowd-sourcing, treating ASR as a transcriber, and using a regression model to predict transcription reliability. Even though the accuracy of individual crowd-workers is only 55% on our data, our best method achieves 90% accuracy on 93% of the utterances, using only 1.3 crowd-worker transcriptions per utterance on average. When forced to transcribe all utterances, our best method matches the accuracy of previous crowdsourcing methods using only one third as many transcriptions.  We also study the effects of various task design factors on transcription latency and accuracy, some of which have not been studied before.},
	att_authors={jw4796, im3247, ta4832, bh2197, jw1957},
	att_categories={C_IIS.11, C_IIS.6, C_IIS.10},
	att_copyright_notice={This version of the work is reprinted here with permission of IEEE for your personal use. Not for redistribution. The definitive version was published in IEEE Workshop on Automatic Speech Recognition and Understanding. {{, 2011-12-11}}
	att_tags={Crowd-sourcing, Transcription, Labeling},
	author={Jason Williams AND Dan Melamed AND Tirso Alonso AND Barbara Hollister AND Jay Wilpon},
	booktitle={{IEEE Workshop on Automatic Speech Recognition and Understanding, Hawaii, USA}},
	institution={{IEEE Workshop on Automatic Speech Recognition and Understanding}},
	title={{Crowd-sourcing for difficult transcription of speech}},