refs.bib

@inproceedings{bernstein_direct_2012,
	location = {New York, {NY}, {USA}},
	title = {Direct Answers for Search Queries in the Long Tail},
	isbn = {978-1-4503-1015-4},
	doi = {10.1145/2207676.2207710},
	series = {{CHI} '12},
	abstract = {Web search engines now offer more than ranked results. Queries on topics like weather, definitions, and movies may return inline results called answers that can resolve a searcher's information need without any additional interaction. Despite the usefulness of answers, they are limited to popular needs because each answer type is manually authored. To extend the reach of answers to thousands of new information needs, we introduce Tail Answers: a large collection of direct answers that are unpopular individually, but together address a large proportion of search traffic. These answers cover long-tail needs such as the average body temperature for a dog, substitutes for molasses, and the keyboard shortcut for a right-click. We introduce a combination of search log mining and paid crowdsourcing techniques to create Tail Answers. A user study with 361 participants suggests that Tail Answers significantly improved users' subjective ratings of search quality and their ability to solve needs without clicking through to a result. Our findings suggest that search engines can be extended to directly respond to a large new class of queries.},
	pages = {237--246},
	booktitle = {Proceedings of the {SIGCHI} Conference on Human Factors in Computing Systems},
	publisher = {{ACM}},
	author = {Bernstein, Michael S. and Teevan, Jaime and Dumais, Susan and Liebling, Daniel and Horvitz, Eric},
	date = {2012},
	keywords = {crowdsourcing, query log analysis, search user interfaces}
}

@inproceedings{teevan_slow_2013,
	location = {New York, {NY}, {USA}},
	title = {Slow Search: Information Retrieval Without Time Constraints},
	isbn = {978-1-4503-2570-7},
	series = {{HCIR} '13},
	shorttitle = {Slow Search},
	abstract = {Significant time and effort has been devoted to reducing the time between query receipt and search engine response, and for good reason. Research suggests that even slightly higher retrieval latency by Web search engines can lead to dramatic decreases in users' perceptions of result quality and engagement with the search results. While users have come to expect rapid responses from search engines, recent advances in our understanding of how people find information suggest that there are scenarios where a search engine could take significantly longer than a fraction of a second to return relevant content. This raises the important question: What would search look like if search engines were not constrained by existing expectations for speed? In this paper, we explore slow search, a class of search where traditional speed requirements are relaxed in favor of a high quality search experience. Via large-scale log analysis and user surveys, we examine how individuals value time when searching. We confirm that speed is important, but also show that there are many search situations where result quality is more important. This highlights intriguing opportunities for search systems to support new search experiences with high quality result content that takes time to identify. Slow search has the potential to change the search experience as we know it.},
	pages = {1:1--1:10},
	booktitle = {Proceedings of the Symposium on Human-Computer Interaction and Information Retrieval},
	publisher = {{ACM}},
	author = {Teevan, Jaime and Collins-Thompson, Kevyn and White, Ryen W. and Dumais, Susan T. and Kim, Yubin},
	date = {2013},
	keywords = {{citeDiss}, crowdsourcing, {INFORMATION} retrieval, Slow search, speed}
}

@report{kim_crowdsourcing_2013,
	location = {Gaithersburg, Maryland},
	title = {Crowdsourcing for Robustness in Web Search},
	type = {Proceedings of {NIST} Special Publication: The Twenty-Second Text {REtrieval} Conference},
	author = {Kim, Yubin and Kevyn, Collins-Thompson and Teevan, Jaime},
	date = {2013-11},
	keywords = {{citeDiss}}
}
@article{downie_music_2003,
	title = {Music information retrieval},
	volume = {37},
	rights = {Copyright © 2002 American Society for Information Science and Technology},
	issn = {1550-8382},
	doi = {10.1002/aris.1440370108},
	pages = {295--340},
	number = {1},
	journaltitle = {Annual Review of Information Science and Technology},
	shortjournal = {Ann. Rev. Info. Sci. Tech.},
	author = {Downie, J. Stephen},
	date = {2003-01-01},
	langid = {english},
}

@article{shera_librarians_1967,
	title = {Librarians against Machines},
	volume = {156},
	issn = {0036-8075, 1095-9203},
	doi = {10.1126/science.156.3776.746},
	pages = {746--750},
	number = {3776},
	journaltitle = {Science},
	shortjournal = {Science},
	author = {Shera, Jesse H.},
	date = {1967-05-12},
	langid = {english},
	pmid = {6022224}
}

@article{lintott_galaxy_2008,
	title = {Galaxy Zoo : Morphologies derived from visual inspection of galaxies from the Sloan Digital Sky Survey},
	shorttitle = {Galaxy Zoo},
	abstract = {In order to understand the formation and subsequent evolution of galaxies one must first distinguish between the two main morphological classes of massive systems: spirals and early-type systems. This paper introduces a project, Galaxy Zoo, which provides visual morphological classifications for nearly one million galaxies, extracted from the Sloan Digital Sky Survey ({SDSS}). This achievement was made possible by inviting the general public to visually inspect and classify these galaxies via the internet. The project has obtained more than 40,000,000 individual classifications made by {\textasciitilde}100,000 participants. We discuss the motivation and strategy for this project, and detail how the classifications were performed and processed. We find that Galaxy Zoo results are consistent with those for subsets of {SDSS} galaxies classified by professional astronomers, thus demonstrating that our data provides a robust morphological catalogue. Obtaining morphologies by direct visual inspection avoids introducing biases associated with proxies for morphology such as colour, concentration or structual parameters. In addition, this catalogue can be used to directly compare {SDSS} morphologies with older data sets. The colour--magnitude diagrams for each morphological class are shown, and we illustrate how these distributions differ from those inferred using colour alone as a proxy for morphology.},
	journaltitle = {0804.4483},
	author = {Lintott, Chris J and Schawinski, Kevin and Slosar, Anze and Land, Kate and Bamford, Steven and Thomas, Daniel and Raddick, M. Jordan and Nichol, Robert C and Szalay, Alex and Andreescu, Dan and Murray, Phil and Berg, Jan van den},
	date = {2008-04-29},
	keywords = {Astrophysics},
	file = {arXiv.org Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\V8EPT2QX\\0804.html:text/html}
}


@book{marshall_future_2000,
	title = {The Future of Annotation in a Digital (Paper) World},
	isbn = {0-87845-107-2},
	abstract = {If order-making in the large is part of the institutional mission of libraries,
then order-making in the small—i.e., the informal work of annotating
and organizing materials collected in the service of particular dayto-
day work or pleasure—is part of the business of library patrons. This
discussion focuses on just such activities; activities that stem from readers'
engagements with texts, and possibly with each other, against a backdrop
of real-world settings and practices. I hesitate to call digital library patrons
users, since that is the word computer scientists tend to use to hide the
characteristics of what we hope is a diverse population.},
	publisher = {Graduate School of Library and Information Science, University of Illinois at Urbana-Champaign.},
	author = {Marshall, C.},
	date = {2000},
	langid = {english},
	file = {Marshall_2000_The Future of Annotation in a Digital (Paper) World.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Graduate School of Library and Information Science, University of Illinois at Urbana-Champaign.2000\\Marshall_2000_The Future of Annotation in a Digital (Paper) World.pdf:application/pdf;Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\HBP3PHXR\\25539.html:text/html}
}



@inproceedings{li_exploring_2008,
	location = {New York, {NY}, {USA}},
	title = {Exploring question subjectivity prediction in community {QA}},
	isbn = {978-1-60558-164-4},
	doi = {10.1145/1390334.1390477},
	series = {{SIGIR} '08},
	abstract = {In this paper we begin to investigate how to automatically determine the subjectivity orientation of questions posted by real users in community question answering ({CQA}) portals. Subjective questions seek answers containing private states, such as personal opinion and experience. In contrast, objective questions request objective, verifiable information, often with support from reliable sources. Knowing the question orientation would be helpful not only for evaluating answers provided by users, but also for guiding the {CQA} engine to process questions more intelligently. Our experiments on Yahoo! Answers data show that our method exhibits promising performance.},
	pages = {735--736},
	booktitle = {Proceedings of the 31st annual international {ACM} {SIGIR} conference on Research and development in information retrieval},
	publisher = {{ACM}},
	author = {Li, Baoli and Liu, Yandong and Ram, Ashwin and Garcia, Ernest V. and Agichtein, Eugene},
	date = {2008},
	keywords = {question classification, subjectivity analysis},
	file = {ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\63GNH6SM\\Li et al. - 2008 - Exploring question subjectivity prediction in comm.pdf:application/pdf}
}



@article{linden_amazon.com_2003,
	title = {Amazon.com recommendations: item-to-item collaborative filtering},
	volume = {7},
	issn = {1089-7801},
	doi = {10.1109/MIC.2003.1167344},
	shorttitle = {Amazon.com recommendations},
	abstract = {Recommendation algorithms are best known for their use on e-commerce Web sites, where they use input about a customer's interests to generate a list of recommended items. Many applications use only the items that customers purchase and explicitly rate to represent their interests, but they can also use other attributes, including items viewed, demographic data, subject interests, and favorite artists. At Amazon.com, we use recommendation algorithms to personalize the online store for each customer. The store radically changes based on customer interests, showing programming titles to a software engineer and baby toys to a new mother. There are three common approaches to solving the recommendation problem: traditional collaborative filtering, cluster models, and search-based methods. Here, we compare these methods with our algorithm, which we call item-to-item collaborative filtering. Unlike traditional collaborative filtering, our algorithm's online computation scales independently of the number of customers and number of items in the product catalog. Our algorithm produces recommendations in real-time, scales to massive data sets, and generates high quality recommendations.},
	pages = {76--80},
	number = {1},
	journaltitle = {{IEEE} Internet Computing},
	author = {Linden, G. and Smith, B. and York, J.},
	date = {2003-01},
	keywords = {Advertising, Aggregates, Amazon.com recommendations, Clustering algorithms, cluster models, Collaboration, customer interests, demographic data, Demography, e-commerce, electronic commerce, Electronic mail, Filtering algorithms, information filtering, information filters, {INFORMATION} retrieval, item-to-item collaborative filtering, massive data sets, online store, Pediatrics, product catalog, real-time, real-time systems, recommendation algorithms, retail data processing, search-based methods, Web sites},
	file = {IEEE Xplore Abstract Record:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\73A5PWAX\\abs_all.html:text/html;IEEE Xplore Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\DDKCZWA6\\Linden et al. - 2003 - Amazon.com recommendations item-to-item collabora.pdf:application/pdf}
}



@inproceedings{vieweg_microblogging_2010,
	title = {Microblogging during two natural hazards events: what twitter may contribute to situational awareness},
	shorttitle = {Microblogging during two natural hazards events},
	pages = {1079--1088},
	booktitle = {Proceedings of the {SIGCHI} conference on human factors in computing systems},
	publisher = {{ACM}},
	author = {Vieweg, Sarah and Hughes, Amanda L. and Starbird, Kate and Palen, Leysia},
	date = {2010},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\RS2ITD4S\\citation.html:text/html;Vieweg et al_2010_Microblogging during two natural hazards events.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2010\\Vieweg et al_2010_Microblogging during two natural hazards events.pdf:application/pdf}
}



@inproceedings{noll_web_2007,
	location = {Berlin, Heidelberg},
	title = {Web Search Personalization via Social Bookmarking and Tagging},
	isbn = {3-540-76297-3 978-3-540-76297-3},
	series = {{ISWC}'07/{ASWC}'07},
	abstract = {In this paper, we present a new approach to web search personalization based on user collaboration and sharing of information about web documents. The proposed personalization technique separates data collection and user profiling from the information system whose contents and indexed documents are being searched for, i.e. the search engines, and uses social bookmarking and tagging to re-rank web search results. It is independent of the search engine being used, so users are free to choose the one they prefer, even if their favorite search engine does not natively support personalization. We show how to design and implement such a system in practice and investigate its feasibility and usefulness with large sets of real-word data and a user study.},
	pages = {367--380},
	booktitle = {Proceedings of the 6th International The Semantic Web and 2Nd Asian Conference on Asian Semantic Web Conference},
	publisher = {Springer-Verlag},
	author = {Noll, Michael G. and Meinel, Christoph},
	date = {2007}
}

@inproceedings{bao_optimizing_2007,
	location = {New York, {NY}, {USA}},
	title = {Optimizing Web Search Using Social Annotations},
	isbn = {978-1-59593-654-7},
	doi = {10.1145/1242572.1242640},
	series = {{WWW} '07},
	abstract = {This paper explores the use of social annotations to improve websearch. Nowadays, many services, e.g. del.icio.us, have been developed for web users to organize and share their favorite webpages on line by using social annotations. We observe that the social annotations can benefit web search in two aspects: 1) the annotations are usually good summaries of corresponding webpages; 2) the count of annotations indicates the popularity of webpages. Two novel algorithms are proposed to incorporate the above information into page ranking: 1) {SocialSimRank} ({SSR})calculates the similarity between social annotations and webqueries; 2) {SocialPageRank} ({SPR}) captures the popularity of webpages. Preliminary experimental results show that {SSR} can find the latent semantic association between queries and annotations, while {SPR} successfully measures the quality (popularity) of a webpage from the web users' perspective. We further evaluate the proposed methods empirically with 50 manually constructed queries and 3000 auto-generated queries on a dataset crawledfrom delicious. Experiments show that both {SSR} and {SPRbenefit} web search significantly.},
	pages = {501--510},
	booktitle = {Proceedings of the 16th International Conference on World Wide Web},
	publisher = {{ACM}},
	author = {Bao, Shenghua and Xue, Guirong and Wu, Xiaoyuan and Yu, Yong and Fei, Ben and Su, Zhong},
	date = {2007},
	keywords = {evaluation, social annotation, social page rank, social similarity, web search}
}

@article{lerman_personalizing_2007,
	title = {Personalizing image search results on flickr},
	journaltitle = {Intelligent Information Personalization},
	author = {Lerman, Kristina and Plangprasopchok, Anon and Wong, Chio},
	date = {2007},
	file = {Lerman et al_2007_Personalizing image search results on flickr.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Intelligent Information Personalization2007\\Lerman et al_2007_Personalizing image search results on flickr.pdf:application/pdf}
}

@article{furnas_vocabulary_1987,
	title = {The vocabulary problem in human-system communication},
	volume = {30},
	issn = {00010782},
	doi = {10.1145/32206.32212},
	pages = {964--971},
	number = {11},
	journaltitle = {Communications of the {ACM}},
	author = {Furnas, G. W. and Landauer, T. K. and Gomez, L. M. and Dumais, S. T.},
	date = {1987-11-01},
	file = {Furnas et al_1987_The vocabulary problem in human-system communication.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Communications of the ACM1987\\Furnas et al_1987_The vocabulary problem in human-system communication.pdf:application/pdf;Shibboleth Authentication Request:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\MXI3X6UN\\login.html:text/html}
}

@article{shu_signing_2012,
	title = {Signing at the beginning makes ethics salient and decreases dishonest self-reports in comparison to signing at the end},
	volume = {109},
	issn = {0027-8424, 1091-6490},
	doi = {10.1073/pnas.1209746109},
	abstract = {Many written forms required by businesses and governments rely on honest reporting. Proof of honest intent is typically provided through signature at the end of, e.g., tax returns or insurance policy forms. Still, people sometimes cheat to advance their financial self-interests—at great costs to society. We test an easy-to-implement method to discourage dishonesty: signing at the beginning rather than at the end of a self-report, thereby reversing the order of the current practice. Using laboratory and field experiments, we find that signing before—rather than after—the opportunity to cheat makes ethics salient when they are needed most and significantly reduces dishonesty.},
	pages = {15197--15200},
	number = {38},
	journaltitle = {Proceedings of the National Academy of Sciences},
	shortjournal = {{PNAS}},
	author = {Shu, Lisa L. and Mazar, Nina and Gino, Francesca and Ariely, Dan and Bazerman, Max H.},
	date = {2012-09-18},
	langid = {english},
	pmid = {22927408},
	keywords = {fraud, morality, nudge, policy-making},
	file = {Shu et al_2012_Signing at the beginning makes ethics salient and decreases dishonest.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Proceedings of the National Academy of Sciences2012\\Shu et al_2012_Signing at the beginning makes ethics salient and decreases dishonest.pdf:application/pdf;Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\I69UWKGU\\15197.html:text/html}
}



@book{csikszentmihalyi_flow_1991,
	title = {Flow: The psychology of optimal experience},
	volume = {41},
	shorttitle = {Flow},
	publisher = {{HarperPerennial} New York},
	author = {Csikszentmihalyi, Mihaly},
	date = {1991}
}


@inproceedings{kazai_crowdsourcing_2011,
	location = {New York, {NY}, {USA}},
	title = {Crowdsourcing for Book Search Evaluation: Impact of Hit Design on Comparative System Ranking},
	isbn = {978-1-4503-0757-4},
	doi = {10.1145/2009916.2009947},
	series = {{SIGIR} '11},
	shorttitle = {Crowdsourcing for Book Search Evaluation},
	abstract = {The evaluation of information retrieval ({IR}) systems over special collections, such as large book repositories, is out of reach of traditional methods that rely upon editorial relevance judgments. Increasingly, the use of crowdsourcing to collect relevance labels has been regarded as a viable alternative that scales with modest costs. However, crowdsourcing suffers from undesirable worker practices and low quality contributions. In this paper we investigate the design and implementation of effective crowdsourcing tasks in the context of book search evaluation. We observe the impact of aspects of the Human Intelligence Task ({HIT}) design on the quality of relevance labels provided by the crowd. We assess the output in terms of label agreement with a gold standard data set and observe the effect of the crowdsourced relevance judgments on the resulting system rankings. This enables us to observe the effect of crowdsourcing on the entire {IR} evaluation process. Using the test set and experimental runs from the {INEX} 2010 Book Track, we find that varying the {HIT} design, and the pooling and document ordering strategies leads to considerable differences in agreement with the gold set labels. We then observe the impact of the crowdsourced relevance label sets on the relative system rankings using four {IR} performance metrics. System rankings based on {MAP} and Bpref remain less affected by different label sets while the Precision@10 and {nDCG}@10 lead to dramatically different system rankings, especially for labels acquired from {HITs} with weaker quality controls. Overall, we find that crowdsourcing can be an effective tool for the evaluation of {IR} systems, provided that care is taken when designing the {HITs}.},
	pages = {205--214},
	booktitle = {Proceedings of the 34th International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Kazai, Gabriella and Kamps, Jaap and Koolen, Marijn and Milic-Frayling, Natasa},
	date = {2011},
	keywords = {book search, {citeDiss}, crowdsourcing quality, prove it},
	file = {Kazai et al_2011_Crowdsourcing for Book Search Evaluation.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2011\\Kazai et al_2011_Crowdsourcing for Book Search Evaluation.pdf:application/pdf}
}



@inproceedings{zhang_human_2012,
	location = {New York, {NY}, {USA}},
	title = {Human computation tasks with global constraints},
	isbn = {978-1-4503-1015-4},
	doi = {10.1145/2207676.2207708},
	series = {{CHI} '12},
	abstract = {An important class of tasks that are underexplored in current human computation systems are complex tasks with global constraints. One example of such a task is itinerary planning, where solutions consist of a sequence of activities that meet requirements specified by the requester. In this paper, we focus on the crowdsourcing of such plans as a case study of constraint-based human computation tasks and introduce a collaborative planning system called Mobi that illustrates a novel crowdware paradigm. Mobi presents a single interface that enables crowd participants to view the current solution context and make appropriate contributions based on current needs. We conduct experiments that explain how Mobi enables a crowd to effectively and collaboratively resolve global constraints, and discuss how the design principles behind Mobi can more generally facilitate a crowd to tackle problems involving global constraints.},
	pages = {217--226},
	booktitle = {Proceedings of the {SIGCHI} Conference on Human Factors in Computing Systems},
	publisher = {{ACM}},
	author = {Zhang, Haoqi and Law, Edith and Miller, Rob and Gajos, Krzysztof and Parkes, David and Horvitz, Eric},
	date = {2012},
	keywords = {collaborative planning, crowdware, groupware, human computation, mixed-initiative interaction},
	file = {ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\U4FIIPZD\\Zhang et al. - 2012 - Human computation tasks with global constraints.pdf:application/pdf}
}


@article{marmorstein_value_1992,
	title = {The Value of Time Spent in Price-Comparison Shopping: Survey and Experimental Evidence},
	volume = {19},
	rights = {Copyright © 1992 Journal of Consumer Research, Inc.},
	issn = {0093-5301},
	shorttitle = {The Value of Time Spent in Price-Comparison Shopping},
	abstract = {The value that consumers place on time spent in price-comparison shopping is central to the economics of information theory and models of consumers' search behavior. Yet few empirical studies have examined consumers' subjective value of time. Building on Gary Becker's work, this article presents two tests of a model of the subjective value of time. In an effort to explain consumers' subjective value of time while they are price-comparison shopping, the model introduces perceived enjoyment of shopping as a new explanatory variable. The findings reveal that respondents incorporate both wage rates and perceived enjoyment of price-comparison shopping into their subjective value of time.},
	pages = {52--61},
	number = {1},
	journaltitle = {Journal of Consumer Research},
	shortjournal = {Journal of Consumer Research},
	author = {Marmorstein, Howard and Grewal, Dhruv and Fishe, Raymond P. H.},
	date = {1992-06-01},
	keywords = {{citeDiss}, temp},
	file = {Marmorstein et al_1992_The Value of Time Spent in Price-Comparison Shopping.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Marmorstein et al_1992_The Value of Time Spent in Price-Comparison Shopping.pdf:application/pdf}
}

@article{konstan_grouplens_1997,
	title = {{GroupLens}: Applying Collaborative Filtering to Usenet News},
	volume = {40},
	issn = {0001-0782},
	doi = {10.1145/245108.245126},
	shorttitle = {{GroupLens}},
	pages = {77--87},
	number = {3},
	journaltitle = {Commun. {ACM}},
	author = {Konstan, Joseph A. and Miller, Bradley N. and Maltz, David and Herlocker, Jonathan L. and Gordon, Lee R. and Riedl, John},
	date = {1997-03},
	keywords = {{citeDiss}, temp},
	file = {ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\K8H65RMZ\\Konstan et al. - 1997 - GroupLens Applying Collaborative Filtering to Use.pdf:application/pdf;ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\W3J2CFHT\\Konstan et al. - 1997 - GroupLens Applying Collaborative Filtering to Use.pdf:application/pdf}
}

@article{gursoy_integrative_2004,
	title = {{AN} {INTEGRATIVE} {MODEL} {OF} {TOURISTS}’ {INFORMATION} {SEARCH} {BEHAVIOR}},
	volume = {31},
	issn = {0160-7383},
	doi = {10.1016/j.annals.2003.12.004},
	abstract = {This article develops a comprehensive theoretical model that integrates the psychological/motivational, economics, and processing approaches into a cohesive whole for understanding tourists’ information seeking behavior. The model proposes that for immediate pre-purchase information needs, a consumer is likely to utilize either internal or external sources, or both. The search is likely to be influenced directly by the perceived internal and external costs, and the level of involvement required. Familiarity and expertise, learning and previous visits indirectly influence the search. Their influences are mediated by familiarity and expertise with the destination, which are, in turn, mediated by external and internal costs. Twenty-one propositions are developed for future testing.},
	pages = {353--373},
	number = {2},
	journaltitle = {Annals of Tourism Research},
	shortjournal = {Annals of Tourism Research},
	author = {Gursoy, Dogan and {McCleary}, Ken W.},
	date = {2004-04},
	keywords = {-clérecherche de renseignements, apprentissage, {citeDiss}, connaissances, engagement, expertise, familiarité, familiarity, information search, involvement, learning, temp},
	file = {Gursoy_McCleary_2004_AN INTEGRATIVE MODEL OF TOURISTS’ INFORMATION SEARCH BEHAVIOR.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Gursoy_McCleary_2004_AN INTEGRATIVE MODEL OF TOURISTS’ INFORMATION SEARCH BEHAVIOR.pdf:application/pdf;ScienceDirect Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\W7FNCH24\\S0160738303001397.html:text/html}
}

@inproceedings{dellarocas_what_2006,
	title = {What motivates consumers to review a product online? A study of the product-specific antecedents of online movie reviews},
	shorttitle = {What motivates consumers to review a product online?},
	booktitle = {{WISE}},
	author = {Dellarocas, Chrysanthos and Narayan, Ritu},
	date = {2006},
	keywords = {{citeDiss}, jdistribution, j-distribution, temp},
	file = {Dellarocas_Narayan_2006_What motivates consumers to review a product online.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2006\\Dellarocas_Narayan_2006_What motivates consumers to review a product online.pdf:application/pdf}
}

@inproceedings{hu_can_2006,
	location = {New York, {NY}, {USA}},
	title = {Can Online Reviews Reveal a Product's True Quality?: Empirical Findings and Analytical Modeling of Online Word-of-mouth Communication},
	isbn = {1-59593-236-4},
	doi = {10.1145/1134707.1134743},
	series = {{EC} '06},
	shorttitle = {Can Online Reviews Reveal a Product's True Quality?},
	abstract = {As a digital version of word-of-mouth, online review has become a major information source for consumers and has very important implications for a wide range of management activities. While some researchers focus their studies on the impact of online product review on sales, an important assumption remains unexamined, that is, can online product review reveal the true quality of the product? To test the validity of this key assumption, this paper first empirically tests the underlying distribution of online reviews with data from Amazon. The results show that 53\% of the products have a bimodal and non-normal distribution. For these products, the average score does not necessarily reveal the product's true quality and may provide misleading recommendations. Then this paper derives an analytical model to explain when the mean can serve as a valid representation of a product's true quality, and discusses its implication on marketing practices.},
	pages = {324--330},
	booktitle = {Proceedings of the 7th {ACM} Conference on Electronic Commerce},
	publisher = {{ACM}},
	author = {Hu, Nan and Pavlou, Paul A. and Zhang, Jennifer},
	date = {2006},
	keywords = {bi-modal distribution, {citeDiss}, online reviews, temp, Word-of-mouth},
	file = {Hu et al_2006_Can Online Reviews Reveal a Product's True Quality.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2006\\Hu et al_2006_Can Online Reviews Reveal a Product's True Quality.pdf:application/pdf}
}

@inproceedings{krishnan_who_2008,
	location = {New York, {NY}, {USA}},
	title = {Who Predicts Better?: Results from an Online Study Comparing Humans and an Online Recommender System},
	isbn = {978-1-60558-093-7},
	doi = {10.1145/1454008.1454042},
	series = {{RecSys} '08},
	shorttitle = {Who Predicts Better?},
	abstract = {Algorithmic recommender systems attempt to predict which items a target user will like based on information about the user's prior preferences and the preferences of a larger community. After more than a decade of widespread use, researchers and system users still debate whether such "impersonal" recommender systems actually perform as well as human recommenders. We compare the performance of {MovieLens} algorithmic predictions with the recommendations made, based on the same user profiles, by active {MovieLens} users. We found that algorithmic collaborative filtering outperformed humans on average, though some individuals outperformed the system substantially and humans on average outperformed the system on certain prediction tasks.},
	pages = {211--218},
	booktitle = {Proceedings of the 2008 {ACM} Conference on Recommender Systems},
	publisher = {{ACM}},
	author = {Krishnan, Vinod and Narayanashetty, Pradeep Kumar and Nathan, Mukesh and Davies, Richard T. and Konstan, Joseph A.},
	date = {2008},
	keywords = {{citeDiss}, human recommenders, mae, movielens, predictions, recommender evaluation, recommender systems, temp},
	file = {ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\ZK2UTR2K\\Krishnan et al. - 2008 - Who Predicts Better Results from an Online Study.pdf:application/pdf}
}

@article{daugherty_exploring_2008,
	title = {Exploring Consumer Motivations for Creating User-Generated Content},
	volume = {8},
	issn = {null},
	doi = {10.1080/15252019.2008.10722139},
	abstract = {The advent of Web 2.0 technologies has enabled the efficient creation and distribution of user-generated content ({UGC}), resulting in vast changes in the online media landscape. For instance, the proliferation of {UGC} has made a strong impact on consumers, media suppliers, and marketing professionals while necessitating research in order to understand both the short and long-term implications of this media content. This exploratory study (n = 325) seeks to investigate consumer consumption and creation of {UGC} and the attitudinal factors that contribute to these actions. The data confirm the established relationship between attitude and behavior and indicate attitude serves as a mediating factor between the use and creation of {UGC}. With regard to the creation of {UGC}, the ego-defensive and social functions of attitude were found to have the most explanatory power.},
	pages = {16--25},
	number = {2},
	journaltitle = {Journal of Interactive Advertising},
	author = {Daugherty, Terry and Eastin, Matthew S. and Bright, Laura},
	date = {2008-03-01},
	keywords = {{citeDiss}, temp},
	file = {Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PUSBFA6T\\Daugherty et al. - 2008 - Exploring Consumer Motivations for Creating User-G.pdf:application/pdf;Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\QDFK6447\\Daugherty et al. - 2008 - Exploring Consumer Motivations for Creating User-G.html:text/html}
}

@movie{pugh_star_2009,
	title = {Star Wars Uncut: Director's Cut},
	url = {www.starwarsuncut.com},
	author = {Pugh, Casey},
	date = {2009},
	keywords = {{citeDiss}, temp}
}

@inproceedings{zhang_enhancing_2009,
	location = {New York, {NY}, {USA}},
	title = {Enhancing Information Scent: Identifying and Recommending Quality Tags},
	isbn = {978-1-60558-500-0},
	doi = {10.1145/1531674.1531676},
	series = {{GROUP} '09},
	shorttitle = {Enhancing Information Scent},
	abstract = {We describe a scenario of tag use and an empirical study of tags as socio-cognitive artifacts providing information scent. We articulated a three-step use scenario of tags, and used it to conceptualize tag "quality" as determined by use. We designed and conducted a user study to explore what attributes of tags and taggers predict the user-rated "quality" of tags. We found that frequency best predicted tag quality, while information entropy provided further refinement. We found that people rated our identified quality tags as higher in quality than general tags. But these identified quality tags were not perceived as better than self-generated tags. We derived a regression model for tag quality and discussed implications for social computing.},
	pages = {1--10},
	booktitle = {Proceedings of the {ACM} 2009 International Conference on Supporting Group Work},
	publisher = {{ACM}},
	author = {Zhang, Shaoke and Farooq, Umer and Carroll, John M.},
	date = {2009},
	keywords = {{citeDiss}, quality tags, sense-making, social bookmarking, temp},
	file = {Zhang et al_2009_Enhancing Information Scent.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2009\\Zhang et al_2009_Enhancing Information Scent.pdf:application/pdf}
}

@inproceedings{bernstein_crowds_2011,
	location = {New York, {NY}, {USA}},
	title = {Crowds in two seconds: enabling realtime crowd-powered interfaces},
	isbn = {978-1-4503-0716-1},
	doi = {10.1145/2047196.2047201},
	series = {{UIST} '11},
	shorttitle = {Crowds in two seconds},
	abstract = {Interactive systems must respond to user input within seconds. Therefore, to create realtime crowd-powered interfaces, we need to dramatically lower crowd latency. In this paper, we introduce the use of synchronous crowds for on-demand, realtime crowdsourcing. With synchronous crowds, systems can dynamically adapt tasks by leveraging the fact that workers are present at the same time. We develop techniques that recruit synchronous crowds in two seconds and use them to execute complex search tasks in ten seconds. The first technique, the retainer model, pays workers a small wage to wait and respond quickly when asked. We offer empirically derived guidelines for a retainer system that is low-cost and produces on-demand crowds in two seconds. Our second technique, rapid refinement, observes early signs of agreement in synchronous crowds and dynamically narrows the search space to focus on promising directions. This approach produces results that, on average, are of more reliable quality and arrive faster than the fastest crowd member working alone. To explore benefits and limitations of these techniques for interaction, we present three applications: Adrenaline, a crowd-powered camera where workers quickly filter a short video down to the best single moment for a photo; and Puppeteer and A{\textbar}B, which examine creative generation tasks, communication with workers, and low-latency voting.},
	pages = {33--42},
	booktitle = {Proceedings of the 24th annual {ACM} symposium on User interface software and technology},
	publisher = {{ACM}},
	author = {Bernstein, Michael S. and Brandt, Joel and Miller, Robert C. and Karger, David R.},
	date = {2011},
	keywords = {{citeDiss}, crowdsourcing, human computation, temp}
}

@inproceedings{tamuz_adaptively_2011,
	title = {Adaptively Learning the Crowd Kernel},
	shorttitle = {Proc. {ICML} 2011},
	booktitle = {Proceedings of the International Conference on Machine Learning},
	author = {Tamuz, Omer and Liu, Ce and Belongie, Serge and Shamir, Ohad and Kalai, Adam Tauman},
	date = {2011},
	keywords = {{citeDiss}, temp},
	file = {[PDF] from icml-2011.org:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PPCZ87MR\\Tamuz et al. - 2011 - Adaptively Learning the Crowd Kernel.pdf:application/pdf}
}

@article{radinsky_learning_2012,
	title = {Learning to Predict from Textual Data},
	volume = {45},
	issn = {1076-9757},
	abstract = {Given a current news event, we tackle the problem of generating plausible predictions of future events it might cause. We present a new methodology for modeling and predicting such future news events using machine learning and data mining techniques. Our Pundit algorithm generalizes examples of causality pairs to infer a causality predictor. To obtain precisely labeled causality examples, we mine 150 years of news articles and apply semantic natural language modeling techniques to headlines containing certain predefined causality patterns. For generalization, the model uses a vast number of world knowledge ontologies. Empirical evaluation on real news articles shows that our Pundit algorithm performs as well as non-expert humans.},
	pages = {641--684},
	number = {1},
	journaltitle = {J. Artif. Int. Res.},
	author = {Radinsky, Kira and Davidovich, Sagie and Markovitch, Shaul},
	date = {2012-09},
	keywords = {{citeDiss}, temp}
}

@inproceedings{alonso_are_2013,
	location = {New York, {NY}, {USA}},
	title = {Are Some Tweets More Interesting Than Others? \#{HardQuestion}},
	isbn = {978-1-4503-2570-7},
	doi = {10.1145/2528394.2528396},
	series = {{HCIR} '13},
	shorttitle = {Are Some Tweets More Interesting Than Others?},
	abstract = {Twitter has evolved into a significant communication nexus, coupling personal and highly contextual utterances with local news, memes, celebrity gossip, headlines, and other microblogging subgenres. If we take Twitter as a large and varied dynamic collection, how can we predict which tweets will be interesting to a broad audience in advance of lagging social indicators of interest such as retweets? The telegraphic form of tweets, coupled with the subjective notion of interestingness, makes it difficult for human judges to agree on which tweets are indeed interesting. In this paper, we address two questions: Can we develop a reliable strategy that results in high-quality labels for a collection of tweets, and can we use this labeled collection to predict a tweet's interestingness? To answer the first question, we performed a series of studies using crowdsourcing to reach a diverse set of workers who served as a proxy for an audience with variable interests and perspectives. This method allowed us to explore different labeling strategies, including varying the judges, the labels they applied, the datasets, and other aspects of the task. To address the second question, we used crowdsourcing to assemble a set of tweets rated as interesting or not; we scored these tweets using textual and contextual features; and we used these scores as inputs to a binary classifier. We were able to achieve moderate agreement (κ = 0.52) between the best classifier and the human assessments, a figure which reflects the challenges of the judgment task.},
	pages = {2:1--2:10},
	booktitle = {Proceedings of the Symposium on Human-Computer Interaction and Information Retrieval},
	publisher = {{ACM}},
	author = {Alonso, Omar and Marshall, Catherine C. and Najork, Marc},
	date = {2013},
	keywords = {{citeDiss}, crowdsourcing, interestingness, label quality, temp, twitter},
	file = {ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\U8KQZV5M\\Alonso et al. - 2013 - Are Some Tweets More Interesting Than Others #Har.pdf:application/pdf}
}

@inproceedings{kokkalis_emailvalet_2013,
	location = {New York, {NY}, {USA}},
	title = {{EmailValet}: managing email overload through private, accountable crowdsourcing},
	isbn = {978-1-4503-1331-5},
	doi = {10.1145/2441776.2441922},
	series = {{CSCW} '13},
	shorttitle = {{EmailValet}},
	abstract = {This paper introduces privacy and accountability techniques for crowd-powered systems. We focus on email task management: tasks are an implicit part of every inbox, but the overwhelming volume of incoming email can bury important requests. We present {EmailValet}, an email client that recruits remote assistants from an expert crowdsourcing marketplace. By annotating each email with its implicit tasks, {EmailValet}'s assistants create a task list that is automatically populated from emails in the user's inbox. The system is an example of a valet approach to crowdsourcing, which aims for parsimony and transparency in access con-trol for the crowd. To maintain privacy, users specify rules that define a sliding-window subset of their inbox that they are willing to share with assistants. To support accountability, {EmailValet} displays the actions that the assistant has taken on each email. In a weeklong field study, participants completed twice as many of their email-based tasks when they had access to crowdsourced assistants, and they became increasingly comfortable sharing their inbox with assistants over time.},
	pages = {1291--1300},
	booktitle = {Proceedings of the 2013 conference on Computer supported cooperative work},
	publisher = {{ACM}},
	author = {Kokkalis, Nicolas and Köhn, Thomas and Pfeiffer, Carl and Chornyi, Dima and Bernstein, Michael S. and Klemmer, Scott R.},
	date = {2013},
	keywords = {access control, {citeDiss}, crowdsourcing, email overload, human assistants, task management, temp}
}

@inproceedings{rzeszotarski_inserting_2013,
	title = {Inserting Micro-Breaks into Crowdsourcing Workflows},
	rights = {Authors who submit to this conference agree to the following terms:   a) \ Authors retain copyright over their work, while allowing the conference to place this unpublished work under a  Creative Commons Attribution License , which allows others to freely access, use, and share the work, with an acknowledgement of the work's authorship and its initial presentation at this conference.   b) \ Authors are able to waive the terms of the {CC} license and enter into separate, additional contractual arrangements for the non-exclusive distribution and subsequent publication of this work (e.g., publish a revised version in a journal, post it to an institutional repository or publish it in a book), with an acknowledgement of its initial presentation at this conference.   c) \ In addition, authors are encouraged to post and share their work online (e.g., in institutional repositories or on their website) at any point before and after the conference.},
	abstract = {Participants in human computation workflows may become fatigued or get bored over long, interminable working hours. This leads to a slump of motivation and morale, which in the long run causes reductions in both productivity and work quality. In this paper we propose an initial investigation into possible ways to alleviate worker fatigue and boredom by employing micro-breaks that provide timely relax to workers during long sequences of tasks. We experimentally test micro-breaks on Amazon’s Mechanical Turk, showing that micro-breaks can significantly improve worker retention rate as task batches reach hours in length, and appear to increase overall worker engagement and commitment to their work.},
	eventtitle = {First {AAAI} Conference on Human Computation and Crowdsourcing},
	booktitle = {First {AAAI} Conference on Human Computation and Crowdsourcing},
	author = {Rzeszotarski, Jeffrey M. and Chi, Ed and Paritosh, Praveen and Dai, Peng},
	date = {2013-03-11},
	langid = {english},
	keywords = {{citeDiss}, temp},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PWNUVD5S\\7544.html:text/html}
}

@article{kiritchenko_sentiment_2014,
	title = {Sentiment Analysis of Short Informal Texts},
	volume = {50},
	issn = {1076-9757},
	abstract = {We describe a state-of-the-art sentiment analysis system that detects (a) the sentiment of short informal textual messages such as tweets and {SMS} (message-level task) and (b) the sentiment of a word or a phrase within a message (term-level task). The system is based on a supervised statistical text classification approach leveraging a variety of surface-form, semantic, and sentiment features. The sentiment features are primarily derived from novel high-coverage tweet-specific sentiment lexicons. These lexicons are automatically generated from tweets with sentiment-word hashtags and from tweets with emoticons. To adequately capture the sentiment of words in negated contexts, a separate sentiment lexicon is generated for negated words. The system ranked first in the {SemEval}-2013 shared task 'Sentiment Analysis in Twitter' (Task 2), obtaining an F-score of 69.02 in the message-level task and 88.93 in the term-level task. Post-competition improvements boost the performance to an F-score of 70.45 (message-level task) and 89.50 (term-level task). The system also obtains state-of-the-art performance on two additional datasets: the {SemEval}-2013 {SMS} test set and a corpus of movie review excerpts. The ablation experiments demonstrate that the use of the automatically generated lexicons results in performance gains of up to 6.5 absolute percentage points.},
	pages = {723--762},
	number = {1},
	journaltitle = {J. Artif. Int. Res.},
	author = {Kiritchenko, Svetlana and Zhu, Xiaodan and Mohammad, Saif M.},
	date = {2014-05},
	keywords = {{citeDiss}, temp}
}

@inproceedings{organisciak_crowd_2014,
	location = {Pittsburgh, {USA}},
	title = {A Crowd of Your Own: Crowdsourcing for On-Demand Personalization},
	series = {{HCOMP} 2014},
	booktitle = {Proceedings of the Second {AAAI} Conference on Human Computation \& Crowdsourcing},
	publisher = {{AAAI}},
	author = {Organisciak, Peter and Teevan, Jaime and Dumais, Susan T. and Miller, Robert C. and Kalai, Adam Tauman},
	date = {2014-11-02},
	keywords = {{citeDiss}, temp}
}

@inproceedings{organisciak_matching_2015,
	location = {Buenos Aires, Argentina},
	title = {Matching and Grokking: Approaches to Personalized Crowdsourcing},
	series = {{IJCAI} '15 (Best Papers from Sister Conferences track)},
	eventtitle = {International Joint Conferences on Artificial Intelligence},
	publisher = {{AAAI}},
	author = {Organisciak, Peter and Teevan, Jaime and Dumais, Susan and Miller, Robert C. and Kalai, Adam Tauman},
	date = {2015-07},
	keywords = {{citeDiss}, temp}
}
@online{chen_mechanical_2012,
	title = {Mechanical Turk Best Practices},
	url = {Mechanical Turk Best Practices},
	type = {Clockwork Raven Wiki},
	author = {Chen, Edwin},
	date = {2012-10-08},
	keywords = {{citeDissProp}}
}

@online{chen_making_2012,
	title = {Making the Most of Mechanical Turk: Tips and Best Practices},
	url = {http://blog.echen.me/2012/04/25/making-the-most-of-mechanical-turk-tips-and-best-practices/},
	type = {Blog},
	author = {Chen, Edwin},
	date = {2012-04-25},
	keywords = {{citeDissProp}}
}

@inproceedings{harris_youre_2011,
	title = {You’re hired! an examination of crowdsourcing incentive models in human resource tasks},
	eventtitle = {{CSDM}},
	pages = {15--18},
	booktitle = {Proceedings of the Workshop on Crowdsourcing for Search and Data Mining ({CSDM}) at the Fourth {ACM} International Conference on Web Search and Data Mining ({WSDM})},
	author = {Harris, Christopher},
	date = {2011},
	keywords = {{citeDiss}},
	file = {Harris_2011_You’re hired.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2011\\Harris_2011_You’re hired.pdf:application/pdf}
}



@article{eisenberg_measuring_1988,
	title = {Measuring relevance judgments},
	volume = {24},
	issn = {0306-4573},
	doi = {10.1016/0306-4573(88)90042-8},
	abstract = {Accurate, reliable measurement of relevance is fundamental to research in information science and to the design, development, and evaluation of information systems. This article describes a study focusing on the measurement of relevance and the application of magnitude estimation, an open-ended scaling technique developed in the field of sensory psychophysics, to the task of measuring relevance judgments. The study found that magnitude estimation is highly appropriate for the measurement of relevance judgments and less influenced by potential biases than commonly used category rating scaling procedures. One biasing factor—a range context effect—was found for both magnitude estimation and category rating scales.},
	pages = {373--389},
	number = {4},
	journaltitle = {Information Processing \& Management},
	shortjournal = {Information Processing \& Management},
	author = {Eisenberg, Michael B.},
	date = {1988},
	file = {ScienceDirect Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\2NQ88B5S\\0306457388900428.html:text/html}
}

@inproceedings{urbano_crowdsourcing_2010,
	title = {Crowdsourcing preference judgments for evaluation of music similarity tasks},
	pages = {9--16},
	booktitle = {{ACM} {SIGIR} workshop on crowdsourcing for search evaluation},
	author = {Urbano, Julián and Morato, Jorge and Marrero, Mónica and Martín, Diego},
	date = {2010},
	file = {[PDF] from monica-marrero.com:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\GBD2QXUR\\Urbano et al. - 2010 - Crowdsourcing preference judgments for evaluation .pdf:application/pdf}
}


@article{typke_ground_2005,
	title = {A Ground Truth For Half A Million Musical Incipits.},
	volume = {3},
	pages = {34--38},
	number = {1},
	journaltitle = {{JDIM}},
	author = {Typke, Rainer and den Hoed, Marc and de Nooijer, Justin and Wiering, Frans and Veltkamp, Remco C.},
	date = {2005},
	file = {[PDF] from computingscience.nl:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\DSMC9DKA\\Typke et al. - 2005 - A Ground Truth For Half A Million Musical Incipits.pdf:application/pdf}
}



@online{hiatt_role_2013,
	title = {The Role of Familiarity, Priming and Perception in Similarity Judgments},
	author = {Hiatt, Laura and Trafton, J. and Trafton, J. and Trafton, J.},
	date = {2013},
	file = {The Role of Familiarity, Priming and Perception in Similarity Judgments | Laboratory for Autonomous Systems Research:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\FVZ5BVXV\\role-familiarity-priming-and-perception-similarity-judgments.html:text/html}
}


@article{golder_usage_2006,
	title = {Usage patterns of collaborative tagging systems},
	volume = {32},
	doi = {10.1177/0165551506062337},
	pages = {198 --208},
	number = {2},
	journaltitle = {Journal of Information Science},
	author = {Golder, Scott A. and Huberman, Bernardo A.},
	date = {2006-04-01},
	keywords = {bookmarks, {citeDiss}, collaborative tagging, Del.icio.us, folksonomy, {ICcited}, sharing, social book-marking, social media, sorted, tagging, web},
}



@incollection{downie_music_2010,
	title = {The music information retrieval evaluation exchange: Some observations and insights},
	pages = {93--115},
	booktitle = {Advances in music information retrieval},
	publisher = {Springer Berlin / Heidelberg},
	author = {Downie, J. Stephen},
	date = {2010}
}


@inproceedings{organisciak_iterative_2012,
	location = {Washington {DC}, {USA}},
	title = {An Iterative Reliability Measure for Semi-anonymous Annotators},
	eventtitle = {Joint Conference on Digital Libraries},
	author = {Organisciak, Peter},
	date = {2012-06},
	keywords = {{hcirCITE}}
}


@inproceedings{wang_managing_2011,
	location = {Utah},
	title = {Managing Crowdsourcing Workers},
	abstract = {The emergence of online crowdsourcing services such as Amazon Mechanical Turk, presents us huge opportunities to distribute micro-tasks at an unprecedented rate and scale. Unfortunately, the high verification cost and the unstable employment relationship give rise to opportunistic behaviors of workers, which in turn exposes the requesters to quality risks. Currently, most requesters rely on redundancy to identify the correct answers. However, existing techniques cannot separate the true (unrecoverable) error rates from the (recoverable) biases that some workers exhibit, which would lead to incorrect assessment of worker quality. Furthermore, massive redundancy is expensive, increasing significantly the cost of crowdsourced solutions. In this paper, we present an algorithm that can easily separate the true error rates from the biases. Also, we describe how to seamlessly integrate the existence of “gold” data for learning the quality of workers. Next, we bring up an approach for actively testing worker quality in order to quicky identify spammers or malicious workers. Finally, we present experimental results to demonstrate the performance of our proposed algorithm.},
	eventtitle = {Winter Conference on Business Intelligence},
	author = {Wang, Jing and Ipeirotis, Panagiotis G., and Provost, Foster},
	date = {2011-03},
	keywords = {bad workers, {hcirCITE}, {hcirMidtermCITE}, opportunistic workers, reliability, trustworthiness, turk}
}

@inproceedings{organisciak_design_2015,
	location = {Newport Beach, {CA}},
	title = {Design Facets of Crowdsourcing},
	series = {{iConference} '15},
	booktitle = {Proceedings of the 2015 {iConference}},
	author = {Organisciak, Peter and Twidale, Michael B.},
	date = {2015-03-24}
}

@book{mcgonigal_reality_2011,
	edition = {Reprint},
	title = {Reality Is Broken: Why Games Make Us Better and How They Can Change the World},
	isbn = {0-14-312061-1},
	shorttitle = {Reality Is Broken},
	pagetotal = {416},
	publisher = {Penguin Books},
	author = {{McGonigal}, Jane},
	date = {2011-12-27}
}

@article{swanson_snapshot_2015,
	title = {Snapshot Serengeti, high-frequency annotated camera trap images of 40 mammalian species in an African savanna},
	volume = {2},
	issn = {2052-4463},
	doi = {10.1038/sdata.2015.26},
	abstract = {Camera traps can be used to address large-scale questions in community ecology by providing systematic data on an array of wide-ranging species. We deployed 225 camera traps across 1,125 km2 in Serengeti National Park, Tanzania, to evaluate spatial and temporal inter-species dynamics. The cameras have operated continuously since 2010 and had accumulated 99,241 camera-trap days and produced 1.2 million sets of pictures by 2013. Members of the general public classified the images via the citizen-science website www.snapshotserengeti.org. Multiple users viewed each image and recorded the species, number of individuals, associated behaviours, and presence of young. Over 28,000 registered users contributed 10.8 million classifications. We applied a simple algorithm to aggregate these individual classifications into a final ‘consensus’ dataset, yielding a final classification for each image and a measure of agreement among individual answers. The consensus classifications and raw imagery provide an unparalleled opportunity to investigate multi-species dynamics in an intact ecosystem and a valuable resource for machine-learning and computer-vision research.},
	journaltitle = {Scientific Data},
	shortjournal = {Sci Data},
	author = {Swanson, Alexandra and Kosmala, Margaret and Lintott, Chris and Simpson, Robert and Smith, Arfon and Packer, Craig},
	date = {2015-06-09},
	pmid = {26097743},
	pmcid = {PMC4460915},
	file = {Swanson et al_2015_Snapshot Serengeti, high-frequency annotated camera trap images of 40 mammalian.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Scientific Data2015\\Swanson et al_2015_Snapshot Serengeti, high-frequency annotated camera trap images of 40 mammalian.pdf:application/pdf}
}

@inproceedings{eveleigh_i_2013,
	title = {I want to be a captain! i want to be a captain!: gamification in the old weather citizen science project},
	shorttitle = {I want to be a captain! i want to be a captain!},
	pages = {79--82},
	booktitle = {Proceedings of the First International Conference on Gameful Design, Research, and Applications},
	publisher = {{ACM}},
	author = {Eveleigh, Alexandra and Jennett, Charlene and Lynn, Stuart and Cox, Anna L.},
	date = {2013},
	file = {Eveleigh et al_2013_I want to be a captain.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2013\\Eveleigh et al_2013_I want to be a captain.pdf:application/pdf;Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\SQAP38A6\\citation.html:text/html}
}

@article{chiesura_introducing_2015,
	title = {Introducing {LibCrowds}: a crowdsourcing platform aimed at enhancing access to British Library collections},
	shorttitle = {Introducing {LibCrowds}},
	author = {Chiesura, Sara and Gallop, Annabel and Mendes, Alex and Mc Gregor, Nora},
	date = {2015-06-08},
	file = {[HTML] from typepad.co.uk:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\EGNBXDVF\\malay.html:text/html}
}



@inproceedings{resnick_grouplens_1994,
	title = {{GroupLens}},
	isbn = {0-89791-689-1},
	doi = {10.1145/192844.192905},
	abstract = {Collaborative filters help people make choices based on the opinions of other people. {GroupLens} is a system for collaborative filtering of netnews, to help people find articles they will like in the huge stream of available articles. News reader clients display predicted scores and make it easy for users to rate articles after they read them. Rating servers, called Better Bit Bureaus, gather and disseminate the ratings. The rating servers predict scores based on the heuristic that people who agreed in the past will probably agree again. Users can protect their privacy by entering ratings under pseudonyms, without reducing the effectiveness of the score prediction. The entire architecture is open: alternative software for news clients and Better Bit Bureaus can be developed independently and can interoperate with the components we have developed.},
	eventtitle = {{CSCW} '94},
	pages = {175--186},
	booktitle = {Proceedings of the 1994 {ACM} conference on Computer supported cooperative work},
	publisher = {{ACM} Press},
	author = {Resnick, Paul and Iacovou, Neophytos and Suchak, Mitesh and Bergstrom, Peter and Riedl, John},
	date = {1994},
	keywords = {candidates, Collaborative filtering},
	file = {Resnick et al. - 1994 - GroupLens.html:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PR8WT5E6\\Resnick et al. - 1994 - GroupLens.html:text/html}
}


@online{gardner_nine_2011,
	title = {Nine Reasons Women Don't Edit Wikipedia (in their own words)},
	abstract = {The New York Times piece on Wikipedia’s gender gap has given rise to dozens of great online conversations about why so few women edit Wikipedia. I've been reading {ALL} of it, because I believe we ne...},
	titleaddon = {Sue Gardner's Blog},
	author = {Gardner, Sue},
	urldate = {2015-08-07},
	date = {2011-02-19},
	keywords = {{citeDiss}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\WI37G4R3\\nine-reasons-why-women-dont-edit-wikipedia-in-their-own-words.html:text/html}
}

@article{wellman_social_2003,
	title = {The Social Affordances of the Internet for Networked Individualism},
	volume = {8},
	issn = {1083-6101},
	doi = {10.1111/j.1083-6101.2003.tb00216.x},
	abstract = {We review the evidence from a number of surveys in which our {NetLab} has been involved about the extent to which the Internet is transforming or enhancing community. The studies show that the Internet is used for connectivity locally as well as globally, although the nature of its use varies in different countries. Internet use is adding on to other forms of communication, rather than replacing them. Internet use is reinforcing the pre-existing turn to societies in the developed world that are organized around networked individualism rather than group or local solidarities. The result has important implications for civic involvement.},
	pages = {0--0},
	number = {3},
	journaltitle = {Journal of Computer-Mediated Communication},
	author = {Wellman, Barry and Quan-Haase, Anabel and Boase, Jeffrey and Chen, Wenhong and Hampton, Keith and Díaz, Isabel and Miyata, Kakuko},
	date = {2003-04-01},
	langid = {english},
	keywords = {{citeDiss}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\85EQRN7K\\abstract.html:text/html}
}

@article{muchnik_origins_2013,
	title = {Origins of power-law degree distribution in the heterogeneity of human activity in social networks},
	volume = {3},
	rights = {© 2013 Macmillan Publishers Limited. All rights reserved},
	doi = {10.1038/srep01783},
	abstract = {The probability distribution of number of ties of an individual in a social network follows a scale-free power-law. However, how this distribution arises has not been conclusively demonstrated in direct analyses of people's actions in social networks. Here, we perform a causal inference analysis and find an underlying cause for this phenomenon. Our analysis indicates that heavy-tailed degree distribution is causally determined by similarly skewed distribution of human activity. Specifically, the degree of an individual is entirely random - following a “maximum entropy attachment” model - except for its mean value which depends deterministically on the volume of the users' activity. This relation cannot be explained by interactive models, like preferential attachment, since the observed actions are not likely to be caused by interactions with other people.},
	journaltitle = {Scientific Reports},
	shortjournal = {Sci. Rep.},
	author = {Muchnik, Lev and Pei, Sen and Parra, Lucas C. and Reis, Saulo D. S. and Andrade Jr, José S. and Havlin, Shlomo and Makse, Hernán A.},
	date = {2013-05-07},
	langid = {english},
	keywords = {Applied physics, {citeDiss}, Complex networks}
}

@inbook{lavrakas_intercoder_2008,
	location = {Thousand Oaks, {CA}, {USA}},
	title = {Intercoder Reliability},
	isbn = {978-1-4129-1808-4 978-1-4129-6394-7},
	booktitle = {Encyclopedia of Survey Research Methods},
	publisher = {{SAGE} Publications, Inc.},
	author = {Cho, Young Ik},
	bookauthor = {Lavrakas, Paul},
	date = {2008},
	keywords = {{citeDiss}}
}

@article{kittur_crowdsourcing_2008,
	title = {Crowdsourcing for usability: Using micro-task markets for rapid, remote, and low-cost user measurements},
	shorttitle = {Crowdsourcing for usability},
	journaltitle = {Proc. {CHI} 2008},
	author = {Kittur, Aniket and Chi, E. and Suh, Bongwon},
	date = {2008},
	keywords = {{citeDiss}}
}

@inproceedings{ambati_towards_2011,
	title = {Towards Task Recommendation in Micro-Task Markets.},
	pages = {1--4},
	booktitle = {Human computation},
	publisher = {Citeseer},
	author = {Ambati, Vamshi and Vogel, Stephan and Carbonell, Jaime G.},
	date = {2011},
	keywords = {{citeDiss}},
	file = {Ambati et al_2011_Towards Task Recommendation in Micro-Task Markets.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Citeseer2011\\Ambati et al_2011_Towards Task Recommendation in Micro-Task Markets.pdf:application/pdf}
}

@misc{_guidelines_2014,
	title = {Guidelines for Academic Requesters},
	url = {http://wiki.wearedynamo.org/index.php/Guidelines_for_Academic_Requesters},
	publisher = {Dynamo Wiki},
	date = {2014},
	keywords = {{citeDiss}}
}

@inproceedings{sen_quest_2007,
	location = {New York, {NY}, {USA}},
	title = {The Quest for Quality Tags},
	isbn = {978-1-59593-845-9},
	doi = {10.1145/1316624.1316678},
	series = {{GROUP} '07},
	abstract = {Many online communities use tags - community selected words or phrases - to help people find what they desire. The quality of tags varies widely, from tags that capture akey dimension of an entity to those that are profane, useless, or unintelligible. Tagging systems must often select a subset of available tags to display to users due to limited screen space. Because users often spread tags they have seen, selecting good tags not only improves an individual's view of tags, it also encourages them to create better tags in the future. We explore implicit (behavioral) and explicit (rating) mechanisms for determining tag quality. Based on 102,056 tag ratings and survey responses collected from 1,039 users over 100 days, we offer simple suggestions to designers of online communities to improve the quality of tags seen by their users.},
	pages = {361--370},
	booktitle = {Proceedings of the 2007 International {ACM} Conference on Supporting Group Work},
	publisher = {{ACM}},
	author = {Sen, Shilad and Harper, F. Maxwell and {LaPitz}, Adam and Riedl, John},
	date = {2007},
	keywords = {{citeDiss}, moderation, tagging, user interfaces},
	file = {Sen et al_2007_The Quest for Quality Tags.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2007\\Sen et al_2007_The Quest for Quality Tags.pdf:application/pdf}
}

@inproceedings{mitra_comparing_2015,
	title = {Comparing Person-and Process-centric Strategies for Obtaining Quality Data on Amazon Mechanical Turk},
	pages = {1345--1354},
	booktitle = {Proceedings of the 33rd Annual {ACM} Conference on Human Factors in Computing Systems},
	publisher = {{ACM}},
	author = {Mitra, Tanushree and Hutto, C. J. and Gilbert, Eric},
	date = {2015},
	keywords = {{citeDiss}},
	file = {Mitra et al_2015_Comparing Person-and Process-centric Strategies for Obtaining Quality Data on.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2015\\Mitra et al_2015_Comparing Person-and Process-centric Strategies for Obtaining Quality Data on.pdf:application/pdf}
}

@article{hofmann_latent_2004,
	title = {Latent Semantic Models for Collaborative Filtering},
	volume = {22},
	issn = {1046-8188},
	doi = {10.1145/963770.963774},
	abstract = {Collaborative filtering aims at learning predictive models of user preferences, interests or behavior from community data, that is, a database of available user preferences. In this article, we describe a new family of model-based algorithms designed for this task. These algorithms rely on a statistical modelling technique that introduces latent class variables in a mixture model setting to discover user communities and prototypical interest profiles. We investigate several variations to deal with discrete and continuous response variables as well as with different objective functions. The main advantages of this technique over standard memory-based methods are higher accuracy, constant time prediction, and an explicit and compact model representation. The latter can also be used to mine for user communitites. The experimental evaluation shows that substantial improvements in accucracy over existing methods and published results can be obtained.},
	pages = {89--115},
	number = {1},
	journaltitle = {{ACM}  Transactions on Information Systems},
	author = {Hofmann, Thomas},
	date = {2004-01},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Hofmann - 2004 - Latent Semantic Models for Collaborative Filtering.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\Hofmann - 2004 - Latent Semantic Models for Collaborative Filtering.pdf:application/pdf}
}

@inproceedings{organisciak_improving_2015,
	location = {Knoxville, {TN}},
	title = {Improving Consistency of Crowdsourced Multimedia Similarity for Evaluation},
	series = {{JCDL} '15},
	eventtitle = {Joint Conference on Digital Libraries 2015},
	publisher = {{ACM}},
	author = {Organisciak, Peter and Downie, J. Stephen},
	date = {2015-06},
	keywords = {{citeDiss}}
}

@inproceedings{lee_survey_2004,
	title = {Survey Of Music Information Needs, Uses, And Seeking Behaviours: Preliminary Findings.},
	volume = {2004},
	shorttitle = {Survey Of Music Information Needs, Uses, And Seeking Behaviours},
	pages = {5th},
	booktitle = {{ISMIR}},
	publisher = {Citeseer},
	author = {Lee, Jin Ha and Downie, J. Stephen},
	date = {2004},
	keywords = {{citeDiss}}
}

@article{polk_rating_2002,
	title = {Rating the similarity of simple perceptual stimuli: asymmetries induced by manipulating exposure frequency},
	volume = {82},
	issn = {0010-0277},
	shorttitle = {Rating the similarity of simple perceptual stimuli},
	abstract = {When judging the similarity of two stimuli, people's ratings often differ depending on the order in which the comparison is presented (A vs. B or B vs. A). Such directional asymmetries have typically been demonstrated using complex concepts that have a large number of semantic features and a standard explanation is that different sets of features are emphasized depending on the direction of the comparison. In this study, we show that directional asymmetries in the similarity of simple perceptual stimuli can be predictably manipulated merely by presenting each member of a pair with different frequency. Participants rated the similarity of color patches before and after performing an irrelevant training task in which a subset of colors was presented ten times more frequently than others. The similarity ratings after training were significantly more asymmetric than the ratings before training. We discuss the implications of these findings for models of similarity judgment and propose a computationally explicit explanation based on asymmetries in representational stability.},
	pages = {B75--88},
	number = {3},
	journaltitle = {Cognition},
	shortjournal = {Cognition},
	author = {Polk, Thad A. and Behensky, Charles and Gonzalez, Richard and Smith, Edward E.},
	date = {2002-01},
	pmid = {11747865},
	keywords = {Adult, attention, {citeDiss}, Color Perception, Concept Formation, Discrimination Learning, Female, Humans, Male, Mental Recall, Neural Networks (Computer)}
}

@article{tversky_features_1977,
	title = {Features of similarity},
	volume = {84},
	rights = {(c) 2012 {APA}, all rights reserved},
	issn = {1939-1471(Electronic);0033-295X(Print)},
	doi = {10.1037/0033-295X.84.4.327},
	abstract = {Questions the metric and dimensional assumptions that underlie the geometric representation of similarity on both theoretical and empirical grounds. A new set-theoretical approach to similarity is developed in which objects are represented as collections of features and similarity is described as a feature-matching process. Specifically, a set of qualitative assumptions is shown to imply the contrast model, which expresses the similarity between objects as a linear combination of the measures of their common and distinctive features. Several predictions of the contrast model are tested in studies of similarity with both semantic and perceptual stimuli. The model is used to uncover, analyze, and explain a variety of empirical phenomena such as the role of common and distinctive features, the relations between judgments of similarity and difference, the presence of asymmetric similarities, and the effects of context on judgments of similarity. The contrast model generalizes standard representations of similarity data in terms of clusters and trees. It is also used to analyze the relations of prototypicality and family resemblance. (39 ref)},
	pages = {327--352},
	number = {4},
	journaltitle = {Psychological Review},
	author = {Tversky, Amos},
	date = {1977},
	keywords = {*Cognitive Processes, *Pattern Discrimination, *Stimulus Discrimination, {citeDiss}, Stimulus Similarity}
}

@article{marsden_interrogating_2012,
	title = {Interrogating Melodic Similarity: A Definitive Phenomenon or the Product of Interpretation?},
	volume = {41},
	issn = {0929-8215},
	doi = {10.1080/09298215.2012.740051},
	shorttitle = {Interrogating Melodic Similarity},
	abstract = {The nature of melodic similarity is interrogated through a survey of the different means by which the phenomenon has been studied, examination of methods for measuring melodic similarity, a Monte Carlo analysis of data from the experiment which formed the basis for the ‘ground truth’ used in the {MIREX} 2005 contest on melodic similarity, and examples of interest in the music of Mozart. Melodic similarity has been studied by a number of means, sometimes quite contrasting, which lead to important differences in the light of the finding that similarity is dependent on context. Models of melodic similarity based on reduction show that the existence of multiple possible reductions forms a natural basis for similarity to depend on interpretation. Examination of the {MIREX} 2005 data shows wide variations in subjects' judgements of melodic similarity and some evidence that the perceived similarity between two melodies can be influenced by the presence of a third melody. Examples from Mozart suggest that he deliberately exploited the possibilities inherent in recognizing similarity through different interpretations. It is therefore proposed that similarity be thought of not as a distinct and definite function of two melodies but as something created in the minds of those who hear the melodies.},
	pages = {323--335},
	number = {4},
	journaltitle = {Journal of New Music Research},
	author = {Marsden, Alan},
	date = {2012-12-01},
	keywords = {{citeDiss}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\33XZXZJG\\09298215.2012.html:text/html}
}

@article{katter_influence_1968,
	title = {The influence of scale form on relevance judgments},
	volume = {4},
	issn = {0020-0271},
	doi = {10.1016/0020-0271(68)90002-8},
	abstract = {This paper reports the results of two studies. The first compared ranking and category rating procedures for measuring relevance of documents to information requirement statements. The comparison measure was number of reversals; the condition where document-requirement pair A is measured as more relevant than pair B by one procedure and as less relevant than pair B by the other. As compared to category rating, ranking produced three times the expected number of reversals. The results are explained in terms of a “cascaded distortion process” that can affect any procedure which arbitrarily restricts distribution shape.

The second study compared the stimulus range and anchoring sensitivities of a nine-point category scale and a magnitude-ratio scale procedure. Results from the category scale were more consistent and more as predicted. Magnitude ratio results were distorted by unrepresentative scale moduli selected by about one-sixth of the judges, a condition which may be correctable. Suggestions for improved anchoring procedures are discussed in light of the findings for the anchoring treatment.},
	pages = {1--11},
	number = {1},
	journaltitle = {Information Storage and Retrieval},
	shortjournal = {Information Storage and Retrieval},
	author = {Katter, R. V.},
	date = {1968-03},
	keywords = {{citeDiss}},
	file = {ScienceDirect Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\QXMCQ3RA\\0020027168900028.html:text/html}
}

@inproceedings{efron_building_2011,
	location = {New Orleans, {USA}},
	title = {Building Topic Models in a Federated Digital Library Through Selective Document Exclusion},
	series = {{ASIS}\&T '11},
	eventtitle = {{ASIS}\&T Annual Meeting},
	booktitle = {Proceedings of the American Society for Information Science and Technology},
	author = {Efron, Miles and Organisciak, Peter and Fenlon, Katrina},
	date = {2011-10},
	keywords = {{citeDiss}, {hcirCITE}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\7R82INVG\\full.html:text/html}
}

@inproceedings{efron_improving_2012,
	location = {New York, {NY}, {USA}},
	title = {Improving Retrieval of Short Texts Through Document Expansion},
	isbn = {978-1-4503-1472-5},
	doi = {10.1145/2348283.2348405},
	series = {{SIGIR} '12},
	abstract = {Collections containing a large number of short documents are becoming increasingly common. As these collections grow in number and size, providing effective retrieval of brief texts presents a significant research problem. We propose a novel approach to improving information retrieval ({IR}) for short texts based on aggressive document expansion. Starting from the hypothesis that short documents tend to be about a single topic, we submit documents as pseudo-queries and analyze the results to learn about the documents themselves. Document expansion helps in this context because short documents yield little in the way of term frequency information. However, as we show, the proposed technique helps us model not only lexical properties, but also temporal properties of documents. We present experimental results using a corpus of microblog (Twitter) data and a corpus of metadata records from a federated digital library. With respect to established baselines, results of these experiments show that applying our proposed document expansion method yields significant improvements in effectiveness. Specifically, our method improves the lexical representation of documents and the ability to let time influence retrieval.},
	pages = {911--920},
	booktitle = {Proceedings of the 35th International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Efron, Miles and Organisciak, Peter and Fenlon, Katrina},
	date = {2012},
	keywords = {{citeDiss}, document expansion, dublin core, {hcirCITE}, {INFORMATION} retrieval, language models, microblogs, temporal {IR}, twitter}
}

@inproceedings{little_turkit_2009,
	location = {New York, {NY}, {USA}},
	title = {{TurKit}: Tools for Iterative Tasks on Mechanical Turk},
	isbn = {978-1-60558-672-4},
	doi = {10.1145/1600150.1600159},
	series = {{HCOMP} '09},
	shorttitle = {{TurKit}},
	abstract = {Mechanical Turk ({MTurk}) is an increasingly popular web service for paying people small rewards to do human computation tasks. Current uses of {MTurk} typically post independent parallel tasks. We are exploring an alternative iterative paradigm, in which workers build on or evaluate each other's work. We describe {TurKit}, a new toolkit for deploying iterative tasks to {MTurk}, with a familiar imperative programming paradigm that effectively uses {MTurk} workers as subroutines.},
	pages = {29--30},
	booktitle = {Proceedings of the {ACM} {SIGKDD} Workshop on Human Computation},
	publisher = {{ACM}},
	author = {Little, Greg and Chilton, Lydia B. and Goldman, Max and Miller, Robert C.},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, human computation, mechanical turk, toolkit}
}

@book{simon_participatory_2010,
	title = {The participatory museum},
	publisher = {Museum 2.0},
	author = {Simon, Nina},
	date = {2010},
	keywords = {{citeDiss}},
	file = {[HTML] from google.com:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\HE9HVAW9\\books.html:text/html}
}

@inproceedings{wiggins_goals_2012,
	title = {Goals and Tasks: Two Typologies of Citizen Science Projects},
	doi = {10.1109/HICSS.2012.295},
	shorttitle = {Goals and Tasks},
	abstract = {Citizen science is a form of research collaboration involving members of the public in scientific research projects to address real-world problems. Often organized as a virtual collaboration, these projects are a type of open movement, with collective goals addressed through open participation in research tasks. We conducted a survey of citizen science projects to elicit multiple aspects of project design and operation. We then clustered projects based on the tasks performed by participants and on the project's stated goals. The clustering results group projects that show similarities along other dimensions, suggesting useful divisions of the projects.},
	eventtitle = {2012 45th Hawaii International Conference on System Science ({HICSS})},
	pages = {3426--3435},
	booktitle = {2012 45th Hawaii International Conference on System Science ({HICSS})},
	author = {Wiggins, A and Crowston, Kevin},
	date = {2012-01},
	keywords = {Approximation methods, {citeDiss}, Citizen science, citizen science project, Collaboration, Communities, Educational institutions, Electronic mail, groupware, Monitoring, open movement, Production, project design, project operation, real-world problem, research collaboration, scientific information systems, scientific research project, virtual collaboration},
	file = {IEEE Xplore Abstract Record:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\NJQZCP74\\Wiggins and Crowston - 2012 - Goals and Tasks Two Typologies of Citizen Science.html:text/html}
}

@article{von_ahn_recaptcha_2008,
	title = {recaptcha: Human-based character recognition via web security measures},
	volume = {321},
	shorttitle = {recaptcha},
	pages = {1465--1468},
	number = {5895},
	journaltitle = {Science},
	author = {von Ahn, Luis and Maurer, Benjamin and {McMillen}, Colin and Abraham, David and Blum, Manuel},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {[HTML] from sciencemag.org:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\NEKH8S8Q\\1465.html:text/html}
}

@article{khatib_algorithm_2011,
	title = {Algorithm discovery by protein folding game players},
	volume = {108},
	pages = {18949--18953},
	number = {47},
	journaltitle = {Proceedings of the National Academy of Sciences},
	author = {Khatib, Firas and Cooper, Seth and Tyka, Michael D. and Xu, Kefan and Makedon, Ilya and Popović, Zoran and Baker, David and Players, Foldit},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {[HTML] from pnas.org:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\EBN8D2SM\\18949.html:text/html;Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\U7W2U2FU\\18949.html:text/html}
}

@book{hotho_information_2006,
	title = {Information retrieval in folksonomies: Search and ranking},
	shorttitle = {Information retrieval in folksonomies},
	publisher = {Springer},
	author = {Hotho, Andreas and Jäschke, Robert and Schmitz, Christoph and Stumme, Gerd},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}, folksonomy, Information Retrieval},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\2JCEKW2A\\11762256_31.html:text/html}
}

@inproceedings{zhou_exploring_2008,
	location = {New York, {NY}, {USA}},
	title = {Exploring Social Annotations for Information Retrieval},
	isbn = {978-1-60558-085-2},
	doi = {10.1145/1367497.1367594},
	series = {{WWW} '08},
	abstract = {Social annotation has gained increasing popularity in many Web-based applications, leading to an emerging research area in text analysis and information retrieval. This paper is concerned with developing probabilistic models and computational algorithms for social annotations. We propose a unified framework to combine the modeling of social annotations with the language modeling-based methods for information retrieval. The proposed approach consists of two steps: (1) discovering topics in the contents and annotations of documents while categorizing the users by domains; and (2) enhancing document and query language models by incorporating user domain interests as well as topical background models. In particular, we propose a new general generative model for social annotations, which is then simplified to a computationally tractable hierarchical Bayesian network. Then we apply smoothing techniques in a risk minimization framework to incorporate the topical information to language models. Experiments are carried out on a real-world annotation data set sampled from del.icio.us. Our results demonstrate significant improvements over the traditional approaches.},
	pages = {715--724},
	booktitle = {Proceedings of the 17th International Conference on World Wide Web},
	publisher = {{ACM}},
	author = {Zhou, Ding and Bian, Jiang and Zheng, Shuyi and Zha, Hongyuan and Giles, C. Lee},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}, folksonomy, {INFORMATION} retrieval, language modeling, social annotations},
	file = {Zhou-et-al_2008_Exploring Social Annotations for Information Retrieval.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2008\\Zhou-et-al_2008_Exploring Social Annotations for Information Retrieval.pdf:application/pdf}
}

@inproceedings{bischoff_can_2008,
	location = {New York, {NY}, {USA}},
	title = {Can All Tags Be Used for Search?},
	isbn = {978-1-59593-991-3},
	doi = {10.1145/1458082.1458112},
	series = {{CIKM} '08},
	abstract = {Collaborative tagging has become an increasingly popular means for sharing and organizing Web resources, leading to a huge amount of user generated metadata. These tags represent quite a few different aspects of the resources they describe and it is not obvious whether and how these tags or subsets of them can be used for search. This paper is the first to present an in-depth study of tagging behavior for very different kinds of resources and systems - Web pages (Del.icio.us), music (Last.fm), and images (Flickr) - and compares the results with anchor text characteristics. We analyze and classify sample tags from these systems, to get an insight into what kinds of tags are used for different resources, and provide statistics on tag distributions in all three tagging environments. Since even relevant tags may not add new information to the search procedure, we also check overlap of tags with content, with metadata assigned by experts and from other sources. We discuss the potential of different kinds of tags for improving search, comparing them with user queries posted to search engines as well as through a user survey. The results are promising and provide more insight into both the use of different kinds of tags for improving search and possible extensions of tagging systems to support the creation of potentially search-relevant tags.},
	pages = {193--202},
	booktitle = {Proceedings of the 17th {ACM} Conference on Information and Knowledge Management},
	publisher = {{ACM}},
	author = {Bischoff, Kerstin and Firan, Claudiu S. and Nejdl, Wolfgang and Paiu, Raluca},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}, collaborative tagging, query classification, tag classification, tagging system analysis and comparison, tag search},
	file = {Bischoff-et-al_2008_Can All Tags Be Used for Search.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2008\\Bischoff-et-al_2008_Can All Tags Be Used for Search.pdf:application/pdf}
}

@inproceedings{heymann_can_2008,
	location = {New York, {NY}, {USA}},
	title = {Can Social Bookmarking Improve Web Search?},
	isbn = {978-1-59593-927-2},
	doi = {10.1145/1341531.1341558},
	series = {{WSDM} '08},
	abstract = {Social bookmarking is a recent phenomenon which has the potential to give us a great deal of data about pages on the web. One major question is whether that data can be used to augment systems like web search. To answer this question, over the past year we have gathered what we believe to be the largest dataset from a social bookmarking site yet analyzed by academic researchers. Our dataset represents about forty million bookmarks from the social bookmarking site del.icio.us. We contribute a characterization of posts to del.icio. us: how many bookmarks exist (about 115 million), how fast is it growing, and how active are the {URLs} being posted about (quite active). We also contribute a characterization of tags used by bookmarkers. We found that certain tags tend to gravitate towards certain domains, and vice versa. We also found that tags occur in over 50 percent of the pages that they annotate, and in only 20 percent of cases do they not occur in the page text, backlink page text, or forward link page text of the pages they annotate. We conclude that social bookmarking can provide search data not currently provided by other sources, though it may currently lack the size and distribution of tags necessary to make a significant impact},
	pages = {195--206},
	booktitle = {Proceedings of the 2008 International Conference on Web Search and Data Mining},
	publisher = {{ACM}},
	author = {Heymann, Paul and Koutrika, Georgia and Garcia-Molina, Hector},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}, collaborative tagging, social bookmarking, web search}
}

@inproceedings{bao_optimizing_2007,
	location = {New York, {NY}, {USA}},
	title = {Optimizing Web Search Using Social Annotations},
	isbn = {978-1-59593-654-7},
	doi = {10.1145/1242572.1242640},
	series = {{WWW} '07},
	abstract = {This paper explores the use of social annotations to improve websearch. Nowadays, many services, e.g. del.icio.us, have been developed for web users to organize and share their favorite webpages on line by using social annotations. We observe that the social annotations can benefit web search in two aspects: 1) the annotations are usually good summaries of corresponding webpages; 2) the count of annotations indicates the popularity of webpages. Two novel algorithms are proposed to incorporate the above information into page ranking: 1) {SocialSimRank} ({SSR})calculates the similarity between social annotations and webqueries; 2) {SocialPageRank} ({SPR}) captures the popularity of webpages. Preliminary experimental results show that {SSR} can find the latent semantic association between queries and annotations, while {SPR} successfully measures the quality (popularity) of a webpage from the web users' perspective. We further evaluate the proposed methods empirically with 50 manually constructed queries and 3000 auto-generated queries on a dataset crawledfrom delicious. Experiments show that both {SSR} and {SPRbenefit} web search significantly.},
	pages = {501--510},
	booktitle = {Proceedings of the 16th International Conference on World Wide Web},
	publisher = {{ACM}},
	author = {Bao, Shenghua and Xue, Guirong and Wu, Xiaoyuan and Yu, Yong and Fei, Ben and Su, Zhong},
	date = {2007},
	keywords = {{citeDiss}, {citeDissProp}, evaluation, social annotation, social page rank, social similarity, web search}
}

@inproceedings{mccreadie_crowdsourcing_2011,
	title = {Crowdsourcing blog track top news judgments at {TREC}},
	pages = {23--26},
	booktitle = {Proceedings of the workshop on crowdsourcing for search and data mining ({CSDM}) at the fourth {ACM} international conference on web search and data mining ({WSDM})},
	author = {{McCreadie}, Richard and Macdonald, Craig and Ounis, Iadh},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {TREC} crowd}
}

@inproceedings{mccreadie_university_2011,
	title = {University of Glasgow at {TREC} 2011: Experiments with Terrier in Crowdsourcing, Microblog, and Web Tracks.},
	shorttitle = {University of Glasgow at {TREC} 2011},
	booktitle = {{TREC}},
	author = {{McCreadie}, Richard and Macdonald, Craig and Santos, Rodrygo {LT} and Ounis, Iadh},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {TREC} crowd}
}

@report{smucker_overview_2012,
	title = {Overview of the trec 2012 crowdsourcing track},
	institution = {{DTIC} Document},
	author = {Smucker, Mark D. and Kazai, Gabriella and Lease, Matthew},
	date = {2012},
	keywords = {{citeDiss}, {citeDissProp}, {TREC} crowd},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\ISWTCKJB\\oai.html:text/html}
}

@article{rouse_preliminary_2010,
	title = {A Preliminary Taxonomy of Crowdsourcing},
	journaltitle = {{ACIS} 2010 Proceedings},
	author = {Rouse, Anne},
	date = {2010-01-01},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, crowdsourcing, Taxonomy}
}

@article{fung_larry_,
	title = {Larry Lessig’s super {PAC} to end super {PACs} raised \$2.5 million in just 2 days. Here’s what comes next.},
	issn = {0190-8286},
	abstract = {The average donation was around \$140, and other numbers.},
	journaltitle = {The Washington Post},
	author = {Fung, Brian},
	langid = {american},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Washington Post Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\6K2WEWH8\\larry-lessigs-super-pac-to-end-super-pacs-raised-2-5-million-in-2-days.html:text/html}
}

@article{cortese_crowdfunding_2013,
	title = {Crowdfunding for Small Business Is Still an Unclear Path},
	issn = {0362-4331},
	url = {http://www.nytimes.com/2013/01/06/business/crowdfunding-for-small-business-is-still-an-unclear-path.html},
	abstract = {For new businesses, the regulatory path to crowdfunding — Internet-based financing that involves ordinary investors — is still far from clear.},
	journaltitle = {The New York Times},
	author = {Cortese, Amy},
	date = {2013-01-05},
	keywords = {Banking and Financial Institutions, {CircleUp} Network Inc, {citeDiss}, {citeDissProp}, Crowdfunding (Internet), Entrepreneurship, Regulation and Deregulation of Industry, Securities and Exchange Commission, Small Business, {SoMoLend} Holdings {LLC}},
	file = {New York Times Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\AU9UVDGM\\crowdfunding-for-small-business-is-still-an-unclear-path.html:text/html}
}

@article{cortese_proposal_2011,
	title = {A Proposal to Allow Small Private Companies to Get Investors Online},
	issn = {0362-4331},
	url = {http://www.nytimes.com/2011/09/26/opinion/a-proposal-to-allow-small-private-companies-to-get-investors-online.html},
	abstract = {A proposal on crowdfunding would make it legal for ordinary investors to put some money (but not enough to bankrupt them) into small, private companies online.},
	journaltitle = {The New York Times},
	author = {Cortese, Amy},
	date = {2011-09-25},
	keywords = {{citeDiss}, {citeDissProp}, Computers and the Internet, Entrepreneurship, Finances, Kickstarter, Kiva.org, {McHenry}, Patrick T, Obama, Barack, Regulation and Deregulation of Industry, Securities and Exchange Commission, Small Business},
	file = {New York Times Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\P9MH2MMJ\\a-proposal-to-allow-small-private-companies-to-get-investors-online.html:text/html}
}

@inproceedings{komarov_crowdsourcing_2013,
	location = {New York, {NY}, {USA}},
	title = {Crowdsourcing Performance Evaluations of User Interfaces},
	isbn = {978-1-4503-1899-0},
	doi = {10.1145/2470654.2470684},
	series = {{CHI} '13},
	abstract = {Online labor markets, such as Amazon's Mechanical Turk ({MTurk}), provide an attractive platform for conducting human subjects experiments because the relative ease of recruitment, low cost, and a diverse pool of potential participants enable larger-scale experimentation and faster experimental revision cycle compared to lab-based settings. However, because the experimenter gives up the direct control over the participants' environments and behavior, concerns about the quality of the data collected in online settings are pervasive. In this paper, we investigate the feasibility of conducting online performance evaluations of user interfaces with anonymous, unsupervised, paid participants recruited via {MTurk}. We implemented three performance experiments to re-evaluate three previously well-studied user interface designs. We conducted each experiment both in lab and online with participants recruited via {MTurk}. The analysis of our results did not yield any evidence of significant or substantial differences in the data collected in the two settings: All statistically significant differences detected in lab were also present on {MTurk} and the effect sizes were similar. In addition, there were no significant differences between the two settings in the raw task completion times, error rates, consistency, or the rates of utilization of the novel interaction mechanisms introduced in the experiments. These results suggest that {MTurk} may be a productive setting for conducting performance evaluations of user interfaces providing a complementary approach to existing methodologies.},
	pages = {207--216},
	booktitle = {Proceedings of the {SIGCHI} Conference on Human Factors in Computing Systems},
	publisher = {{ACM}},
	author = {Komarov, Steven and Reinecke, Katharina and Gajos, Krzysztof Z.},
	date = {2013},
	keywords = {{citeDiss}, {citeDissProp}, crowdsourcing, mechanical turk, user interface evaluation}
}

@inproceedings{yilmaz_simple_2008,
	location = {New York, {NY}, {USA}},
	title = {A Simple and Efficient Sampling Method for Estimating {AP} and {NDCG}},
	isbn = {978-1-60558-164-4},
	doi = {10.1145/1390334.1390437},
	series = {{SIGIR} '08},
	abstract = {We consider the problem of large scale retrieval evaluation. Recently two methods based on random sampling were proposed as a solution to the extensive effort required to judge tens of thousands of documents. While the first method proposed by Aslam et al. [1] is quite accurate and efficient, it is overly complex, making it difficult to be used by the community, and while the second method proposed by Yilmaz et al., {infAP} [14], is relatively simple, it is less efficient than the former since it employs uniform random sampling from the set of complete judgments. Further, none of these methods provide confidence intervals on the estimated values. The contribution of this paper is threefold: (1) we derive confidence intervals for {infAP}, (2) we extend {infAP} to incorporate nonrandom relevance judgments by employing stratified random sampling, hence combining the efficiency of stratification with the simplicity of random sampling, (3) we describe how this approach can be utilized to estimate {nDCG} from incomplete judgments. We validate the proposed methods using {TREC} data and demonstrate that these new methods can be used to incorporate nonrandom samples, as were available in {TREC} Terabyte track '06.},
	pages = {603--610},
	booktitle = {Proceedings of the 31st Annual International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Yilmaz, Emine and Kanoulas, Evangelos and Aslam, Javed A.},
	date = {2008},
	keywords = {average precision, {citeDiss}, {citeDissProp}, evaluation, incomplete judgments, {infAP}, {nDCG}, sampling}
}

@article{maslow_theory_1943,
	title = {A theory of human motivation},
	volume = {50},
	rights = {(c) 2012 {APA}, all rights reserved},
	issn = {1939-1471(Electronic);0033-295X(Print)},
	doi = {10.1037/h0054346},
	abstract = {After listing the propositions that must be considered as basic, the author formulates a theory of human motivation in line with these propositions and with the known facts derived from observation and experiment. There are 5 sets of goals (basic needs) which are related to each other and are arranged in a hierarchy of prepotency. When the most prepotent goal is realized, the next higher need emerges. "Thus man is a perpetually wanting animal." Thwarting, actual or imminent, of these basic needs provides a psychological threat that leads to psychopathy.},
	pages = {370--396},
	number = {4},
	journaltitle = {Psychological Review},
	author = {Maslow, A.H.},
	date = {1943},
	keywords = {*Motivation, Antisocial Personality Disorder, {citeDiss}, {citeDissProp}, {citeiConf}14, Human Development}
}

@article{alderfer_empirical_1969,
	title = {An empirical test of a new theory of human needs},
	volume = {4},
	pages = {142--175},
	number = {2},
	journaltitle = {Organizational behavior and human performance},
	author = {Alderfer, Clayton P.},
	date = {1969},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14},
	file = {[HTML] from sciencedirect.com:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\2WFJHRP3\\003050736990004X.html:text/html}
}

@article{sanger_fate_2009,
	title = {The Fate of Expertise after Wikipedia},
	volume = {6},
	doi = {10.3366/E1742360008000543},
	pages = {52--73},
	number = {1},
	journaltitle = {Episteme},
	author = {Sanger, Lawrence M.},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14},
	file = {Cambridge Journals Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\T8JHP7G4\\displayAbstract.html:text/html}
}

@article{angwin_volunteers_2009,
	title = {Volunteers log off as Wikipedia ages},
	volume = {23},
	journaltitle = {Wall Street Journal},
	author = {Angwin, Julia and Fowler, Geoffrey A.},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14}
}

@online{_about_,
	title = {About Pinterest},
	url = {http://about.pinterest.com/en},
	titleaddon = {Pinterest},
	urldate = {2014-08-15},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\CJEDEUWD\\en.html:text/html}
}

@article{thompson_if_2008,
	title = {If You Liked This, You’re Sure to Love That},
	issn = {0362-4331},
	url = {http://www.nytimes.com/2008/11/23/magazine/23Netflix-t.html},
	abstract = {Basement hackers and amateur mathematicians are competing to improve the program that Netflix uses to recommend {DVDs} — and to win \$1 million in the process.},
	journaltitle = {The New York Times},
	author = {Thompson, Clive},
	date = {2008-11-23},
	keywords = {{citeDiss}, {citeDissProp}, Computers and the Internet, {DVD} (Digital Versatile Disc), Motion Pictures, Netflix Incorporated, Ratings and Rating Systems, Retail Stores and Trade},
	file = {New York Times Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\3IBHT4N3\\23Netflix-t.html:text/html}
}

@inreference{_wikipedia:size_2014,
	title = {Wikipedia:Size of Wikipedia},
	rights = {Creative Commons Attribution-{ShareAlike} License},
	url = {http://en.wikipedia.org/w/index.php?title=Wikipedia:Size_of_Wikipedia&oldid=615924147},
	shorttitle = {Wikipedia},
	abstract = {This Wikipedia:Statistics page measures the size of the English-language edition of Wikipedia; mostly page and article count. There are currently 4,579,708 articles in the English Wikipedia.},
	booktitle = {Wikipedia, the free encyclopedia},
	date = {2014-08-13},
	langid = {english},
	note = {Page Version {ID}: 615924147},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\7RGJ2NUT\\index.html:text/html}
}

@inproceedings{agichtein_improving_2006,
	location = {New York, {NY}, {USA}},
	title = {Improving Web Search Ranking by Incorporating User Behavior Information},
	isbn = {1-59593-369-7},
	doi = {10.1145/1148170.1148177},
	series = {{SIGIR} '06},
	abstract = {We show that incorporating user behavior data can significantly improve ordering of top results in real web search setting. We examine alternatives for incorporating feedback into the ranking process and explore the contributions of user feedback compared to other common web search features. We report results of a large scale evaluation over 3,000 queries and 12 million user interactions with a popular web search engine. We show that incorporating implicit feedback can augment other features, improving the accuracy of a competitive web search ranking algorithms by as much as 31\% relative to the original performance.},
	pages = {19--26},
	booktitle = {Proceedings of the 29th Annual International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Agichtein, Eugene and Brill, Eric and Dumais, Susan},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}, implicit relevance feedback, web search, web search ranking}
}

@online{taylor_friendfeed_2007,
	title = {{FriendFeed} Blog: I like it, I like it},
	url = {http://blog.friendfeed.com/2007/10/i-like-it-i-like-it.html},
	shorttitle = {{FriendFeed} Blog},
	titleaddon = {friendblog},
	author = {Taylor, Bret},
	urldate = {2014-08-09},
	date = {2007-10-30},
	keywords = {{citeDiss}, {citeDissProp}, friendfeed, ratings, rating scales},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\HMFGS4ZS\\i-like-it-i-like-it.html:text/html}
}

@book{abt_serious_1987,
	title = {Serious Games},
	isbn = {978-0-8191-6148-2},
	abstract = {The author explores the ways in which games can be used to instruct and inform as well as provide pleasure. He uses innovative approaches to problem solving through individualized game techniques. Topics include: improving education with games; educational games for the physical and social sciences; games for the learning disadvantaged; games for occupational choice and training; games for planning and problem solving in government and industry; and the future of serious games. This book was originally published in 1970 by Viking Press.},
	pagetotal = {200},
	publisher = {University Press of America},
	author = {Abt, Clark C.},
	date = {1987-01-01},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}, Education / Experimental Methods, gamification, Psychology / Psychotherapy / Counseling, serious games}
}

@book{michael_serious_2005,
	title = {Serious Games: Games That Educate, Train, and Inform},
	isbn = {1-59200-622-1},
	shorttitle = {Serious Games},
	abstract = {Coverage includes- David "{RM}" Michael has been a successful independent software developer for over 10 years, working in a variety of industries, including video games. He is the owner of {DavidRM} Software (www.davidrm.com) and co-owner of Samu Games (www.samugames.com). Michael is the author of The Indie Game Development Survival Guide, and his articles about game design, development, and the game development industry have appeared on {GameDev}.net (www.gamedev.net) and in the book Game Design Perspectives. His blog about independent games, serious games, and independent software is Joe Indie (www.joeindie.com). Sande Chen has been active in the gaming industry for over five years. She has written for mainstream and industry publications, including Secrets of the Game Business, and was a speaker at the 2005 Game Developers Conference. Her past game credits include Independent Games Festival winner Terminus, Scooby-Doo, and {JamDat} Scrabble. Chen holds dual degrees in economics and in writing and humanistic studies from the Massachusetts Institute of Technology, an M.Sc. in economics from the London School of Economics, and an M.F.A. in cinema-television from the University of Southern California. In 1996, she was nominated for a Grammy in music video direction. She currently works as a freelance writer/game designer. Covers techniques to make entertainment-oriented games richer and provide a deeper experience. The focus on serious games continues to grow--from coverage in the media to conferences and buzz within the game development community. Provides an overview of the major markets for serious games, including current examples and future anticipation.},
	publisher = {Muska \& Lipman/Premier-Trade},
	author = {Michael, David R. and Chen, Sandra L.},
	date = {2005},
	keywords = {{citeDiss}, {citeDissProp}, gamification, serious games}
}

@book{ritterfeld_serious_2010,
	title = {Serious Games: Mechanisms and Effects},
	isbn = {978-1-135-84891-0},
	shorttitle = {Serious Games},
	pagetotal = {553},
	publisher = {Routledge},
	author = {Ritterfeld, Ute and Cody, Michael and Vorderer, Peter},
	date = {2010-04-26},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}, Education / Computers \& Technology, Games / Video \& Electronic, gamification, Language Arts \& Disciplines / Communication Studies, serious games, Social Science / Media Studies}
}

@book{twain_adventures_1920,
	title = {The Adventures of Tom Sawyer},
	abstract = {The book that introduced the world to the iconic American characters of Tom Sawyer and Huckleberry Finn, this 1876 novel by Mark Twain follows the mischievous exploits of the two young boys, who find themselves in situations both humorous and dangerous. Never short of ways to stir up trouble in his hometown on the Mississippi River, Tom uses his wits to get both in and out of tight spots, often with Huck at his side. Featuring moments of significant social commentary, these interconnected tales essentially served as a dry run for Twain's notably weightier sequel, Adventures of Huckleberry Finn.},
	pagetotal = {330},
	publisher = {Harper \& brothers},
	author = {Twain, Mark},
	date = {1920},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}}
}

@misc{_requester_2011,
	title = {Requester Best Practices},
	url = {http://mturkpublic.s3.amazonaws.com/docs/MTURK_BP.pdf},
	abstract = {Amazon Mechanical Turk is a marketplace for work where businesses (aka Requesters) publish tasks (aka 
{HITS}), and human providers (aka Workers) complete them. Amazon Mechanical Turk gives businesses 
immediate access to a diverse, global, on-demand, scalable workforce and gives Workers a selection of 
thousands of tasks to complete whenever and wherever it's convenient. 
There are many ways to structure your work in Mechanical Turk. This guide helps you optimize your 
approach to using Mechanical Turk to get the most accurate results at the best price with the turnaround 
time your business needs. Use this guide as you plan, design, and test your Amazon Mechanical Turk 
{HITs}},
	publisher = {Amazon Web Services {LLC}},
	urldate = {2014-08-08},
	date = {2011-06},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inproceedings{ponte_language_1998,
	location = {New York, {NY}, {USA}},
	title = {A Language Modeling Approach to Information Retrieval},
	isbn = {1-58113-015-5},
	doi = {10.1145/290941.291008},
	series = {{SIGIR} '98},
	pages = {275--281},
	booktitle = {Proceedings of the 21st Annual International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Ponte, Jay M. and Croft, W. Bruce},
	date = {1998},
	keywords = {candidates, {citeDiss}, {citeDissProp}}
}

@inproceedings{song_general_1999,
	location = {New York, {NY}, {USA}},
	title = {A General Language Model for Information Retrieval},
	isbn = {1-58113-146-1},
	doi = {10.1145/319950.320022},
	series = {{CIKM} '99},
	abstract = {Statistical language modeling has been successfully used for speech recognition, part-of-speech tagging, and syntactic parsing. Recently, it has also been applied to information retrieval. According to this new paradigm, each document is viewed as a language sample, and a query as a generation process. The retrieved documents are ranked based on the probabilities of producing a query from the corresponding language models of these documents. In this paper, we will present a new language model for information retrieval, which is based on a range of data smoothing techniques, including the Good-Turning estimate, curve-fitting functions, and model combinations. Our model is conceptually simple and intuitive, and can be easily extended to incorporate probabilities of phrases such as word pairs and word triples. The experiments with the Wall Street Journal and {TREC}4 data sets showed that the performance of our model is comparable to that of {INQUERY} and better than that of another language model for information retrieval. In particular, word pairs are shown to be useful in improving the retrieval performance.},
	pages = {316--321},
	booktitle = {Proceedings of the Eighth International Conference on Information and Knowledge Management},
	publisher = {{ACM}},
	author = {Song, Fei and Croft, W. Bruce},
	date = {1999},
	keywords = {{citeDiss}, {citeDissProp}, curve-fitting functions, good-turing estimate, model combinations, statistical language modeling}
}

@inproceedings{dong_time_2010,
	location = {New York, {NY}, {USA}},
	title = {Time is of the Essence: Improving Recency Ranking Using Twitter Data},
	isbn = {978-1-60558-799-8},
	doi = {10.1145/1772690.1772725},
	series = {{WWW} '10},
	shorttitle = {Time is of the Essence},
	abstract = {Realtime web search refers to the retrieval of very fresh content which is in high demand. An effective portal web search engine must support a variety of search needs, including realtime web search. However, supporting realtime web search introduces two challenges not encountered in non-realtime web search: quickly crawling relevant content and ranking documents with impoverished link and click information. In this paper, we advocate the use of realtime micro-blogging data for addressing both of these problems. We propose a method to use the micro-blogging data stream to detect fresh {URLs}. We also use micro-blogging data to compute novel and effective features for ranking fresh {URLs}. We demonstrate these methods improve effective of the portal web search engine for realtime web search.},
	pages = {331--340},
	booktitle = {Proceedings of the 19th International Conference on World Wide Web},
	publisher = {{ACM}},
	author = {Dong, Anlei and Zhang, Ruiqiang and Kolari, Pranam and Bai, Jing and Diaz, Fernando and Chang, Yi and Zheng, Zhaohui and Zha, Hongyuan},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}, recency modeling, recency ranking, twitter}
}

@inproceedings{zhai_model-based_2001,
	location = {New York, {NY}, {USA}},
	title = {Model-based Feedback in the Language Modeling Approach to Information Retrieval},
	isbn = {1-58113-436-3},
	doi = {10.1145/502585.502654},
	series = {{CIKM} '01},
	abstract = {The language modeling approach to retrieval has been shown to perform well empirically. One advantage of this new approach is its statistical foundations. However, feedback, as one important component in a retrieval system, has only been dealt with heuristically in this new retrieval approach: the original query is usually literally expanded by adding additional terms to it. Such expansion-based feedback creates an inconsistent interpretation of the original and the expanded query. In this paper, we present a more principled approach to feedback in the language modeling approach. Specifically, we treat feedback as updating the query language model based on the extra evidence carried by the feedback documents. Such a model-based feedback strategy easily fits into an extension of the language modeling approach. We propose and evaluate two different approaches to updating a query language model based on feedback documents, one based on a generative probabilistic model of feedback documents and one based on minimization of the {KL}-divergence over feedback documents. Experiment results show that both approaches are effective and outperform the Rocchio feedback approach.},
	pages = {403--410},
	booktitle = {Proceedings of the Tenth International Conference on Information and Knowledge Management},
	publisher = {{ACM}},
	author = {Zhai, Chengxiang and Lafferty, John},
	date = {2001},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{von_hippel_democratizing_2006,
	title = {Democratizing Innovation},
	author = {von Hippel, Eric},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\6N5NBVDI\\0262720477.html:text/html}
}

@report{von_hippel_sources_1988,
	location = {Rochester, {NY}},
	title = {The Sources of Innovation},
	abstract = {Presents a series of studies showing that the sources of innovation vary greatly; possible sources include innovation users, suppliers of innovation-related components, and product manufacturers. These types of roles are known as functional areas. Specific areas of innovation are marked by having innovators predominantly in one specific functional area. Using empirical data from industrial histories, the authors show that this innovation-function relationship has held in scientific instrument, semiconductor and printed circuit board assembly process innovations. Users are predominantly the innovators in these fields. Also identifies a few industries where manufacturers are typically the innovators and a few others where suppliers tend to be.        Analysis of the economic rents of innovation expected by potential innovators can often, if not always, by itself predict the functional source of innovation. Innovating firms will do so only when these rents prove attractive. Two factors suggest that this will tend to limit exploitation of the innovation to a functional area. First, it is difficult for innovators to adopt new functional relationships to their innovations. Second, innovators face a poor ability to capture innovation rents by licensing their innovation-related knowledge to others. This hypothesis and its implications are tested against the empirical datasets used initially. The role of informal R\&D know-how trading is also discussed and analyzed using the Prisoner's Dilemma. Guidance is given to innovation managers and policymakers.  ({CAR})},
	number = {{ID} 1496218},
	institution = {Social Science Research Network},
	type = {{SSRN} Scholarly Paper},
	author = {von Hippel, Eric},
	date = {1988},
	keywords = {{citeDiss}, {citeDissProp}, Economic rents, Industrial research, Information behavior, Information exchange, Innovation management, Innovation policies, Innovation process, Know-how, Licensing strategies, Manufacturing firms, R\&D, Suppliers, User needs},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\TZQ5Q426\\login.html:text/html}
}

@misc{norvig_english_,
	title = {English Letter Frequency Counts: Mayzner Revisited or {ETAOIN} {SRHLDCU}},
	url = {http://norvig.com/mayzner.html},
	shorttitle = {English Letter Frequency Counts},
	author = {Norvig, Peter},
	urldate = {2014-05-20},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{downie_music_2006,
	title = {The Music Information Retrieval Evaluation {eXchange} ({MIREX})},
	volume = {12},
	pages = {795--825},
	number = {12},
	journaltitle = {D-Lib Magazine},
	author = {Downie, J. Stephen},
	date = {2006},
	keywords = {{citeDiss}}
}

@article{koren_bellkor_2009,
	title = {The bellkor solution to the netflix grand prize},
	journaltitle = {Netflix prize documentation},
	author = {Koren, Yehuda},
	date = {2009},
	keywords = {{citeDiss}},
	file = {[PDF] from osu.edu:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\8W56GMFB\\Koren - 2009 - The bellkor solution to the netflix grand prize.pdf:application/pdf}
}

@inproceedings{gruzd_evalutron_2007,
	location = {New York, {NY}, {USA}},
	title = {Evalutron 6000: Collecting Music Relevance Judgments},
	isbn = {978-1-59593-644-8},
	doi = {10.1145/1255175.1255307},
	series = {{JCDL} '07},
	shorttitle = {Evalutron 6000},
	pages = {507--507},
	booktitle = {Proceedings of the 7th {ACM}/{IEEE}-{CS} Joint Conference on Digital Libraries},
	publisher = {{ACM}},
	author = {Gruzd, Anatoliy A. and Downie, J. Stephen and Jones, M. Cameron and Lee, Jin Ha},
	date = {2007},
	keywords = {{citeDiss}, {MIREX}, music digital libraries, music information retrieval, music similarity}
}

@inproceedings{urbano_audio_2011,
	title = {Audio Music Similarity and Retrieval: Evaluation Power and Stability.},
	shorttitle = {Audio Music Similarity and Retrieval},
	pages = {597--602},
	booktitle = {{ISMIR}},
	author = {Urbano, Julián and Martín, Diego and Marrero, Mónica and Morato, Jorge},
	date = {2011},
	keywords = {{citeDiss}},
	file = {[PDF] from julian-urbano.info:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\JR6CEFGB\\Urbano et al. - 2011 - Audio Music Similarity and Retrieval Evaluation P.pdf:application/pdf}
}

@inproceedings{lee_crowdsourcing_2010,
	title = {Crowdsourcing Music Similarity Judgments using Mechanical Turk.},
	pages = {183--188},
	booktitle = {{ISMIR}},
	author = {Lee, Jin Ha},
	date = {2010},
	keywords = {{citeDiss}},
	file = {[PDF] from ismir.net:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\8ZB68KB4\\Lee - 2010 - Crowdsourcing Music Similarity Judgments using Mec.pdf:application/pdf}
}

@online{howe_birth_2006,
	title = {Birth of a Meme},
	url = {http://www.crowdsourcing.com/cs/2006/05/birth_of_a_meme.html},
	titleaddon = {Crowdsourcing},
	author = {Howe, Jeff},
	urldate = {2014-04-26},
	date = {2006-05-27},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{lamere_social_2008,
	title = {Social tagging and music information retrieval},
	volume = {37},
	pages = {101--114},
	number = {2},
	journaltitle = {Journal of New Music Research},
	author = {Lamere, Paul},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {[PDF] from vigliensoni.com:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\UX77S4RS\\Lamere - 2008 - Social tagging and music information retrieval.pdf:application/pdf;Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\39XEFUJW\\09298210802479284.html:text/html}
}

@article{harris_applying_2012,
	title = {Applying human computation mechanisms to information retrieval},
	volume = {49},
	issn = {1550-8390},
	doi = {10.1002/meet.14504901050},
	abstract = {Crowdsourcing and Games with a Purpose ({GWAP}) have each received considerable attention in recent years. These two human computation mechanisms assist with tasks that cannot be solved by computers alone. Despite this increased attention, much of this transformation has been limited to a few aspects of Information Retrieval ({IR}). In this paper, we examine these two mechanisms' applicability to {IR}. Using an {IR} model, we apply criteria to determine the suitability of these crowdsourcing and {GWAP} mechanisms to each step of the model. Our analysis illustrates that these mechanisms can apply to several of these steps with good returns.},
	pages = {1--10},
	number = {1},
	journaltitle = {Proceedings of the American Society for Information Science and Technology},
	shortjournal = {Proc. Am. Soc. Info. Sci. Tech.},
	author = {Harris, Christopher G. and Srinivasan, Padmini},
	date = {2012-01-01},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}, crowdsourcing, human computation, Information Retrieval}
}

@book{kraut_building_2011,
	location = {Cambridge, {MA}},
	title = {Building Successful Online Communities},
	publisher = {{MIT} Press},
	author = {Kraut, Robert E. and Resnick, Paul},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, crowdsourcing}
}

@article{ryan_intrinsic_2000,
	title = {Intrinsic and Extrinsic Motivations: Classic Definitions and New Directions},
	volume = {25},
	issn = {0361-476X},
	doi = {10.1006/ceps.1999.1020},
	shorttitle = {Intrinsic and Extrinsic Motivations},
	abstract = {Intrinsic and extrinsic types of motivation have been widely studied, and the distinction between them has shed important light on both developmental and educational practices. In this review we revisit the classic definitions of intrinsic and extrinsic motivation in light of contemporary research and theory. Intrinsic motivation remains an important construct, reflecting the natural human propensity to learn and assimilate. However, extrinsic motivation is argued to vary considerably in its relative autonomy and thus can either reflect external control or true self-regulation. The relations of both classes of motives to basic human needs for autonomy, competence and relatedness are discussed.},
	pages = {54--67},
	number = {1},
	journaltitle = {Contemporary Educational Psychology},
	shortjournal = {Contemporary Educational Psychology},
	author = {Ryan, Richard M. and Deci, Edward L.},
	date = {2000-01},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, extrinsic motivation, Intrinsic motivation, motivation},
	file = {ScienceDirect Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\QUNSM9WG\\S0361476X99910202.html:text/html}
}

@article{zwass_cocreation_2010,
	title = {Co-Creation: Toward a Taxonomy and an Integrated Research Perspective},
	volume = {15},
	doi = {10.2753/JEC1086-4415150101},
	pages = {11--48},
	number = {1},
	journaltitle = {International Journal of Electronic Commerce},
	author = {Zwass, Vladamir},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, cocreation, crowdsourcing, Taxonomy},
	file = {Zwass_2010_Co-Creation.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\International Journal of Electronic Commerce2010\\Zwass_2010_Co-Creation.pdf:application/pdf}
}

@inproceedings{schenk_crowdsourcing_2009,
	title = {Crowdsourcing: What can be Outsourced to the Crowd, and Why?},
	shorttitle = {Crowdsourcing},
	booktitle = {Workshop on Open Source Innovation, Strasbourg, France},
	author = {Schenk, Eric and Guittard, Claude},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, crowdsourcing, Taxonomy},
	file = {Schenk_Guittard_2009_Crowdsourcing.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2009\\Schenk_Guittard_2009_Crowdsourcing.pdf:application/pdf}
}

@incollection{vukovic_towards_2010,
	title = {Towards a Research Agenda for Enterprise Crowdsourcing},
	rights = {Springer Berlin Heidelberg},
	isbn = {978-3-642-16557-3 978-3-642-16558-0},
	series = {Lecture Notes in Computer Science},
	abstract = {Over the past few years the crowdsourcing paradigm has evolved from its humble beginnings as isolated purpose-built initiatives, such as Wikipedia and Elance and Mechanical Turk to a growth industry employing over 2 million knowledge workers, contributing over half a billion dollars to the digital economy. Web 2.0 provides the technological foundations upon which the crowdsourcing paradigm evolves and operates, enabling networked experts to work collaboratively to complete a specific task. Enterprise crowdsourcing poses interesting challenges for both academic and industrial research along the social, legal, and technological dimensions. In this paper we describe the challenges that researchers and practitioners face when thinking about various aspects of enterprise crowdsourcing. First, to establish technological foundations, what are the interaction models and protocols between the Enterprise and the crowd. Secondly, how is crowdsourcing going to face the challenges in quality assurance, enabling Enterprises to optimally leverage the scalable workforce. Thirdly, what are the novel (Web) applications enabled by Enterprise crowdsourcing.},
	pages = {425--434},
	number = {6415},
	booktitle = {Leveraging Applications of Formal Methods, Verification, and Validation},
	publisher = {Springer Berlin Heidelberg},
	author = {Vukovic, Maja and Bartolini, Claudio},
	editor = {Margaria, Tiziana and Steffen, Bernhard},
	date = {2010-01-01},
	keywords = {business process modeling, {citeDiss}, {citeDissProp}, {citeiConf}14, Computer Communication Networks, crowdsourcing, Data Mining and Knowledge Discovery, Information Systems Applications (incl.Internet), Logics and Meanings of Programs, Programming Languages, Compilers, Interpreters, Software Engineering},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\A4VHV7IM\\978-3-642-16558-0_36.html:text/html}
}

@inproceedings{eickhoff_quality_2012,
	location = {New York, {NY}, {USA}},
	title = {Quality Through Flow and Immersion: Gamifying Crowdsourced Relevance Assessments},
	isbn = {978-1-4503-1472-5},
	doi = {10.1145/2348283.2348400},
	series = {{SIGIR} '12},
	shorttitle = {Quality Through Flow and Immersion},
	abstract = {Crowdsourcing is a market of steadily-growing importance upon which both academia and industry increasingly rely. However, this market appears to be inherently infested with a significant share of malicious workers who try to maximise their profits through cheating or sloppiness. This serves to undermine the very merits crowdsourcing has come to represent. Based on previous experience as well as psychological insights, we propose the use of a game in order to attract and retain a larger share of reliable workers to frequently-requested crowdsourcing tasks such as relevance assessments and clustering. In a large-scale comparative study conducted using recent {TREC} data, we investigate the performance of traditional {HIT} designs and a game-based alternative that is able to achieve high quality at significantly lower pay rates, facing fewer malicious submissions.},
	pages = {871--880},
	booktitle = {Proceedings of the 35th International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Eickhoff, Carsten and Harris, Christopher G. and de Vries, Arjen P. and Srinivasan, Padmini},
	date = {2012},
	keywords = {{citeDiss}, {citeDissProp}, clustering, crowdsourcing, gamification, relevance assessments, serious games}
}

@incollection{alonso_design_2011,
	title = {Design and Implementation of Relevance Assessments Using Crowdsourcing},
	rights = {Springer Berlin Heidelberg},
	isbn = {978-3-642-20160-8 978-3-642-20161-5},
	series = {Lecture Notes in Computer Science},
	abstract = {In the last years crowdsourcing has emerged as a viable platform for conducting relevance assessments. The main reason behind this trend is that makes possible to conduct experiments extremely fast, with good results and at low cost. However, like in any experiment, there are several details that would make an experiment work or fail. To gather useful results, user interface guidelines, inter-agreement metrics, and justification analysis are important aspects of a successful crowdsourcing experiment. In this work we explore the design and execution of relevance judgments using Amazon Mechanical Turk as crowdsourcing platform, introducing a methodology for crowdsourcing relevance assessments and the results of a series of experiments using {TREC} 8 with a fixed budget. Our findings indicate that workers are as good as {TREC} experts, even providing detailed feedback for certain query-document pairs. We also explore the importance of document design and presentation when performing relevance assessment tasks. Finally, we show our methodology at work with several examples that are interesting in their own.},
	pages = {153--164},
	number = {6611},
	booktitle = {Advances in Information Retrieval},
	publisher = {Springer Berlin Heidelberg},
	author = {Alonso, Omar and Baeza-Yates, Ricardo},
	editor = {Clough, Paul and Foley, Colum and Gurrin, Cathal and Jones, Gareth J. F. and Kraaij, Wessel and Lee, Hyowon and Mudoch, Vanessa},
	date = {2011-01-01},
	keywords = {Artificial Intelligence (incl. Robotics), {citeDiss}, {citeDissProp}, Database Management, Data Mining and Knowledge Discovery, design, Information Storage and Retrieval, Information Systems Applications (incl.Internet), Multimedia Information Systems},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\A6MZIU6B\\978-3-642-20161-5_16.html:text/html}
}

@article{bell_bellkor_2008,
	title = {The {BellKor} 2008 Solution to the Netflix Prize},
	journaltitle = {Statistics Research Department at {AT}\&T Research},
	author = {Bell, Robert M. and Koren, Yehuda and Volinsky, Chris},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inproceedings{organisciak_personalized_2013,
	location = {Palm Spring, {CA}},
	title = {Personalized Human Computation},
	eventtitle = {{HCOMP} 2013},
	author = {Organisciak, Peter and Teevan, Jaime and Dumais, Susan and Miller, Robert C. and Kalai, Adam Tauman},
	date = {2013},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inreference{_bias_,
	title = {bias, adj., n., and adv.},
	booktitle = {{OED} Online},
	publisher = {Oxford University Press},
	urldate = {2014-01-09},
	langid = {british},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {OED snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\WV62CVIG\\bias, adj., n., and adv..html:text/html}
}

@book{neuendorf_content_2002,
	location = {Thousand Oaks, {CA}, {USA}},
	title = {The Content Analysis Guidebook},
	pagetotal = {301},
	publisher = {Sage Publications},
	author = {Neuendorf, Kimberly A.},
	date = {2002},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inproceedings{trant_investigating_2006,
	title = {Investigating social tagging and folksonomy in art museums with steve. museum},
	booktitle = {Proceedings of the {WWW}’06 Collaborative Web Tagging Workshop},
	author = {Trant, Jennifer and Wyman, Bruce},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Trant_Wyman_2006_Investigating social tagging and folksonomy in art museums with steve.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2006\\Trant_Wyman_2006_Investigating social tagging and folksonomy in art museums with steve.pdf:application/pdf}
}

@inproceedings{organisciak_incidental_2013,
	location = {Lincoln, Nebraska},
	title = {Incidental Crowdsourcing: Crowdsourcing in the Periphery},
	abstract = {As the customs of the Internet grow increasingly collaborative, crowdsourcing offers an appealing frame for looking at the interaction of users with online systems and each other. However, it is a broad term that fails to emphasize the use of crowds in subtler system augmentation.

This paper introduces incidental crowdsourcing ({IC}): an approach to user-provided item description that adopts crowdsourcing as a frame for thinking about augmentative features of system design. {IC} is intended to frame discussion around peripheral and non-critical system design choices.

A provisional definition of incidental crowdsourcing will be defined in this paper, and then refined based on examples seen in practice. {IC} will be examined from both the user and system ends, positioned within existing work, and considered in the context of its benefits and drawbacks. This approach allows us to explore the robustness and feasibility of {IC}, looking at the implications inherent to accepting the provisional definition.

The consequences of considering system design on a scale between {IC} and non-{IC} design choices remain to be seen. Toward this goal, the second part of this paper shows a study comparing the participation habits of users in two online systems — one that is representative of {IC} properties and one that is not. This study finds differences in user engagement between the two systems.},
	eventtitle = {Digital Humanities 2013},
	author = {Organisciak, Peter},
	date = {2013-07-17},
	keywords = {{citeDiss}, {citeDissProp}}
}

@online{chen_improving_2013,
	title = {Improving Twitter search with real-time human computation},
	url = {https://blog.twitter.com/2013/improving-twitter-search-real-time-human-computation},
	abstract = {One of the magical things about Twitter is that it opens a window to the world in real-time. An event happens, and seconds later, people share it across the planet. Consider, for example, what happ......},
	titleaddon = {Twitter Engineering Blog},
	author = {Chen, Edwin and Jain, Alpa},
	date = {2013-01-08},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\NWJQ7G5V\\improving-twitter-search-real-time-human-computation.html:text/html}
}

@inproceedings{efron_hashtag_2010,
	location = {New York, {NY}, {USA}},
	title = {Hashtag Retrieval in a Microblogging Environment},
	isbn = {978-1-4503-0153-4},
	doi = {10.1145/1835449.1835616},
	series = {{SIGIR} '10},
	abstract = {Microblog services let users broadcast brief textual messages to people who "follow" their activity. Often these posts contain terms called hashtags, markers of a post's meaning, audience, etc. This poster treats the following problem: given a user's stated topical interest, retrieve useful hashtags from microblog posts. Our premise is that a user interested in topic x might like to find hashtags that are often applied to posts about x. This poster proposes a language modeling approach to hashtag retrieval. The main contribution is a novel method of relevance feedback based on hashtags. The approach is tested on a corpus of data harvested from twitter.com.},
	pages = {787--788},
	booktitle = {Proceedings of the 33rd International {ACM} {SIGIR} Conference on Research and Development in Information Retrieval},
	publisher = {{ACM}},
	author = {Efron, Miles},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}, hashtag, microblog, relevance feedback, twitter}
}

@inproceedings{shiells_generating_2010,
	location = {New York, {NY}, {USA}},
	title = {Generating Document Summaries from User Annotations},
	isbn = {978-1-4503-0372-9},
	doi = {10.1145/1871962.1871978},
	series = {{ESAIR} '10},
	abstract = {In this paper we analyze tweets that share the same link as another form of social annotation. By extracting all tweets that contain the same link over a period of time, we can generate a summary of what the crowd is writing about that particular link.},
	pages = {25--26},
	booktitle = {Proceedings of the Third Workshop on Exploiting Semantic Annotations in Information Retrieval},
	publisher = {{ACM}},
	author = {Shiells, Karen and Alonso, Omar and Lee, Ho John},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}, crowdsourcing, social search, summarization, twitter}
}

@inproceedings{finin_annotating_2010,
	location = {Stroudsburg, {PA}, {USA}},
	title = {Annotating Named Entities in Twitter Data with Crowdsourcing},
	series = {{CSLDAMT} '10},
	abstract = {We describe our experience using both Amazon Mechanical Turk ({MTurk}) and Crowd-Flower to collect simple named entity annotations for Twitter status updates. Unlike most genres that have traditionally been the focus of named entity experiments, Twitter is far more informal and abbreviated. The collected annotations and annotation techniques will provide a first step towards the full study of named entity recognition in domains like Facebook and Twitter. We also briefly describe how to use {MTurk} to collect judgements on the quality of "word clouds."},
	pages = {80--88},
	booktitle = {Proceedings of the {NAACL} {HLT} 2010 Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk},
	publisher = {Association for Computational Linguistics},
	author = {Finin, Tim and Murnane, Will and Karandikar, Anand and Keller, Nicholas and Martineau, Justin and Dredze, Mark},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}, crowdsourcing, mechanical turk, paid crowdsourcing, twitter}
}

@inproceedings{organisciak_evaluating_2012,
	location = {Baltimore, {MD}},
	title = {Evaluating rater quality and rating difficulty in online annotation activities},
	volume = {49},
	doi = {10.1002/meet.14504901166},
	series = {{ASIS}\&T '12},
	abstract = {Gathering annotations from non-expert online raters is an attractive method for quickly completing large-scale annotation tasks, but the increased possibility of unreliable annotators and diminished work quality remains a cause for concern. In the context of information retrieval, where human-encoded relevance judgments underlie the evaluation of new systems and methods, the ability to quickly and reliably collect trustworthy annotations allows for quicker development and iteration of research.In the context of paid online workers, this study evaluates indicators of non-expert performance along three lines: temporality, experience, and agreement. It is found that user performance is a key indicator for future performance. Additionally, the time spent by raters familiarizing themselves with a new set of tasks is important for rater quality, as is long-term familiarity with a topic being rated.These findings may inform large-scale digital collections' use of non-expert raters for performing more purposive and affordable online annotation activities.},
	eventtitle = {{ASIS}\&T},
	pages = {1--10},
	booktitle = {Proceedings of the American Society for Information Science and Technology},
	author = {Organisciak, Peter and Efron, Miles and Fenlon, Katrina and Senseney, Megan},
	date = {2012},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\KMHITP8J\\abstract\;jsessionid=D2B4003B525D768D514C414B7C85E315.html:text/html}
}

@article{galton_vox_1907,
	title = {Vox populi},
	volume = {75},
	pages = {450--451},
	journaltitle = {Nature},
	author = {Galton, Francis},
	date = {1907},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\AJHAXVIM\\1907Natur..75..html:text/html}
}

@article{law_human_2011,
	title = {Human Computation},
	volume = {5},
	issn = {1939-4608, 1939-4616},
	doi = {10.2200/S00371ED1V01Y201107AIM013},
	pages = {1--121},
	number = {3},
	journaltitle = {Synthesis Lectures on Artificial Intelligence and Machine Learning},
	author = {Law, Edith and von Ahn, Luis},
	date = {2011-06-30},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Morgan & Claypool Publishers - Synthesis Lectures on Artificial Intelligence and Machine Learning - 5(3)\:1 - Abstract:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\BTJCHEXW\\S00371ED1V01Y201107AIM013.html:text/html;Morgan & Claypool Publishers - Synthesis Lectures on Artificial Intelligence and Machine Learning - 5(3)\:1 - Abstract:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\CCKE2ZTG\\S00371ED1V01Y201107AIM013.html:text/html}
}

@inproceedings{springer_for_2008,
	title = {For the common good: The Library of Congress Flickr pilot project},
	shorttitle = {For the common good},
	author = {Springer, Michelle and Dulabahn, Beth and Michel, Phil and Natanson, Barbara and Reser, David W. and Ellison, Nicole B. and Zinkham, Helena and Woodward, David},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14},
	file = {[PDF] from loc.gov:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\ESPKMZX7\\Prints et al. - 2008 - For the common good The Library of Congress Flick.pdf:application/pdf;Springer et al_2008_For the common good.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2008\\Springer et al_2008_For the common good.pdf:application/pdf}
}

@article{causer_transcription_2012,
	title = {Transcription maximized; expense minimized? Crowdsourcing and editing The Collected Works of Jeremy Bentham},
	volume = {27},
	issn = {0268-1145, 1477-4615},
	doi = {10.1093/llc/fqs004},
	shorttitle = {Transcription maximized; expense minimized?},
	abstract = {This article discusses the crowdsourced manuscript transcription project Transcribe Bentham, and how it will impact upon long-established editorial practices at the Bentham Project, University College London, which is producing the new and authoritative edition of The Collected Works of Jeremy Bentham. We site Transcribe Bentham in the burgeoning field of scholarly crowdsourcing projects, and, by detailing our experiences of running and administering the project, attempt to assess the potential benefits of engaging the public in humanities research. The article examines the conceptualization and development of Transcribe Bentham, and how editorial practices at the Bentham Project may change as a result. We account for the design of the bespoke transcription tool which is at the project's heart, and which allows volunteers to transcribe the material and encode it in {TEI}-compliant {XML}. We attempt to answer five key questions: is crowdsourcing the transcription of complex manuscripts cost-effective? Is crowdsourcing exploitative? Are volunteer-produced transcripts of sufficient quality for editorial use and uploading to a digital repository, and what quality controls are required? Does crowdsourcing ensure sustainability and widen access to this priceless material? And finally, should the success of a project like Transcribe Bentham be measured solely according to cost-effectiveness or the volume of work produced, or do considerations of public engagement and access outweigh such concerns?},
	pages = {119--137},
	number = {2},
	journaltitle = {Literary and Linguistic Computing},
	shortjournal = {Lit Linguist Computing},
	author = {Causer, Tim and Tonra, Justin and Wallace, Valerie},
	date = {2012-06-01},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\SI2TC3DD\\119.html:text/html}
}

@book{raymond_cathedral_1999,
	title = {The Cathedral and the Bazaar},
	abstract = {I anatomize a successful open-source project, fetchmail, that was run as a deliberate test of the surprising theories about software engineering suggested by the history of Linux. I discuss these theories in terms of two fundamentally different development styles, the ``cathedral'' model of most of the commercial world versus the ``bazaar'' model of the Linux world. I show that these models derive from opposing assumptions about the nature of the software-debugging task. I then make a sustained argument from the Linux experience for the proposition that ``Given enough eyeballs, all bugs are shallow'', suggest productive analogies with other self-correcting systems of selfish agents, and conclude with some exploration of the implications of this insight for the future of software.},
	pagetotal = {241},
	publisher = {O'Reilly Media},
	author = {Raymond, Eric S.},
	date = {1999},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{efron_information_2011,
	title = {Information search and retrieval in microblogs},
	volume = {62},
	issn = {1532-2890},
	doi = {10.1002/asi.21512},
	abstract = {Modern information retrieval ({IR}) has come to terms with numerous new media in efforts to help people find information in increasingly diverse settings. Among these new media are so-called microblogs. A microblog is a stream of text that is written by an author over time. It comprises many very brief updates that are presented to the microblog's readers in reverse-chronological order. Today, the service called Twitter is the most popular microblogging platform. Although microblogging is increasingly popular, methods for organizing and providing access to microblog data are still new. This review offers an introduction to the problems that face researchers and developers of {IR} systems in microblog settings. After an overview of microblogs and the behavior surrounding them, the review describes established problems in microblog retrieval, such as entity search and sentiment analysis, and modeling abstractions, such as authority and quality. The review also treats user-created metadata that often appear in microblogs. Because the problem of microblog search is so new, the review concludes with a discussion of particularly pressing research issues yet to be studied in the field.},
	pages = {996--1008},
	number = {6},
	journaltitle = {Journal of the American Society for Information Science and Technology},
	author = {Efron, Miles},
	date = {2011},
	langid = {english},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inproceedings{bigham_vizwiz_2010,
	location = {New York, {NY}, {USA}},
	title = {{VizWiz}: nearly real-time answers to visual questions},
	isbn = {978-1-4503-0271-5},
	doi = {10.1145/1866029.1866080},
	series = {{UIST} '10},
	shorttitle = {{VizWiz}},
	abstract = {The lack of access to visual information like text labels, icons, and colors can cause frustration and decrease independence for blind people. Current access technology uses automatic approaches to address some problems in this space, but the technology is error-prone, limited in scope, and quite expensive. In this paper, we introduce {VizWiz}, a talking application for mobile phones that offers a new alternative to answering visual questions in nearly real-time - asking multiple people on the web. To support answering questions quickly, we introduce a general approach for intelligently recruiting human workers in advance called {quikTurkit} so that workers are available when new questions arrive. A field deployment with 11 blind participants illustrates that blind people can effectively use {VizWiz} to cheaply answer questions in their everyday lives, highlighting issues that automatic approaches will need to address to be useful. Finally, we illustrate the potential of using {VizWiz} as part of the participatory design of advanced tools by using it to build and evaluate {VizWiz}::{LocateIt}, an interactive mobile tool that helps blind people solve general visual search problems.},
	pages = {333--342},
	booktitle = {Proceedings of the 23nd annual {ACM} symposium on User interface software and technology},
	publisher = {{ACM}},
	author = {Bigham, Jeffrey P. and Jayant, Chandrika and Ji, Hanjie and Little, Greg and Miller, Andrew and Miller, Robert C. and Miller, Robin and Tatarowicz, Aubrey and White, Brandyn and White, Samual and Yeh, Tom},
	date = {2010},
	keywords = {blind users, {citeDiss}, non-visual interfaces, real-time human computation}
}

@inproceedings{geiger_managing_2011,
	title = {Managing the crowd: towards a taxonomy of crowdsourcing processes},
	shorttitle = {Managing the crowd},
	pages = {1--15},
	booktitle = {Proceedings of the seventeenth Americas conference on information systems, Detroit, Michigan},
	author = {Geiger, David and Seedorf, Stefan and Schulze, Thimo and Nickerson, Robert and Schader, Martin},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14},
	file = {Geiger-et-al_2011_Managing the crowd.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2011\\Geiger-et-al_2011_Managing the crowd.pdf:application/pdf}
}

@online{wales_insist_2006,
	title = {Insist on Sources},
	url = {http://lists.wikimedia.org/pipermail/wikien-l/2006-July/050773.html},
	titleaddon = {{WikiEN}-l},
	author = {Wales, Jimmy},
	date = {2006-07-19},
	keywords = {{citeDiss}, {citeDissProp}}
}

@report{page_pagerank_1999,
	title = {The {PageRank} Citation Ranking: Bringing Order to the Web.},
	abstract = {The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes {PageRank}, a mathod for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare {PageRank} to an idealized random Web surfer. We show how to efficiently compute {PageRank} for large numbers of pages. And, we show how to apply {PageRank} to search and to user navigation.},
	number = {1999-66},
	institution = {Stanford {InfoLab}},
	type = {Technical Report},
	author = {Page, Lawrence and Brin, Sergey and Motwani, Rajeev and Winograd, Terry},
	date = {1999-11},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14}
}

@inproceedings{wei_lda-based_2006,
	location = {New York, {NY}, {USA}},
	title = {{LDA}-based document models for ad-hoc retrieval},
	isbn = {1-59593-369-7},
	doi = {10.1145/1148170.1148204},
	series = {{SIGIR} '06},
	abstract = {Search algorithms incorporating some form of topic model have a long history in information retrieval. For example, cluster-based retrieval has been studied since the 60s and has recently produced good results in the language model framework. An approach to building topic models based on a formal generative model of documents, Latent Dirichlet Allocation ({LDA}), is heavily cited in the machine learning literature, but its feasibility and effectiveness in information retrieval is mostly unknown. In this paper, we study how to efficiently use {LDA} to improve ad-hoc retrieval. We propose an {LDA}-based document model within the language modeling framework, and evaluate it on several {TREC} collections. Gibbs sampling is employed to conduct approximate inference in {LDA} and the computational complexity is analyzed. We show that improvements over retrieval using cluster-based models can be obtained with reasonable efficiency.},
	pages = {178--185},
	booktitle = {Proceedings of the 29th annual international {ACM} {SIGIR} conference on Research and development in information retrieval},
	publisher = {{ACM}},
	author = {Wei, Xing and Croft, W. Bruce},
	date = {2006},
	keywords = {{citeDiss}, document model, {INFORMATION} retrieval, language model, latent dirichlet allocation ({LDA}), topic model},
	file = {ACM Full Text PDF:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\DCZNMFPD\\Wei and Croft - 2006 - LDA-based document models for ad-hoc retrieval.pdf:application/pdf}
}

@inproceedings{liu_cluster-based_2004,
	location = {New York, {NY}, {USA}},
	title = {Cluster-based retrieval using language models},
	isbn = {1-58113-881-4},
	doi = {10.1145/1008992.1009026},
	series = {{SIGIR} '04},
	abstract = {Previous research on cluster-based retrieval has been inconclusive as to whether it does bring improved retrieval effectiveness over document-based retrieval. Recent developments in the language modeling approach to {IR} have motivated us to re-examine this problem within this new retrieval framework. We propose two new models for cluster-based retrieval and evaluate them on several {TREC} collections. We show that cluster-based retrieval can perform consistently across collections of realistic size, and significant improvements over document-based retrieval can be obtained in a fully automatic manner and without relevance information provided by human.},
	pages = {186--193},
	booktitle = {Proceedings of the 27th annual international {ACM} {SIGIR} conference on Research and development in information retrieval},
	publisher = {{ACM}},
	author = {Liu, Xiaoyong and Croft, W. Bruce},
	date = {2004},
	keywords = {{citeDiss}, {citeDissProp}, cluster-based language model, cluster-based retrieval, cluster model, hierarchical clustering, {INFORMATION} retrieval, language model, query-specific clustering, smoothing, static clustering, topic model}
}

@article{holley_crowdsourcing_2010,
	title = {Crowdsourcing: How and Why Should Libraries Do It?},
	volume = {16},
	issn = {1082-9873},
	doi = {10.1045/march2010-holley},
	shorttitle = {Crowdsourcing},
	number = {3},
	journaltitle = {D-Lib Magazine},
	author = {Holley, Rose},
	date = {2010-03},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inproceedings{zhai_study_2001,
	location = {New York, {NY}, {USA}},
	title = {A study of smoothing methods for language models applied to Ad Hoc information retrieval},
	isbn = {1-58113-331-6},
	doi = {10.1145/383952.384019},
	series = {{SIGIR} '01},
	abstract = {Language modeling approaches to information retrieval are attractive and promising because they connect the problem of retrieval with that of language model estimation, which has been studied extensively in other application areas such as speech recognition. The basic idea of these approaches is to estimate a language model for each document, and then rank documents by the likelihood of the query according to the estimated language model. A core problem in language model estimation is smoothing, which adjusts the maximum likelihood estimator so as to correct the inaccuracy due to data sparseness. In this paper, we study the problem of language model smoothing and its influence on retrieval performance.  We examine the sensitivity of retrieval performance to the smoothing parameters and compare several popular smoothing methods on different test collections.},
	pages = {334--342},
	booktitle = {Proceedings of the 24th annual international {ACM} {SIGIR} conference on Research and development in information retrieval},
	publisher = {{ACM}},
	author = {Zhai, Chengxiang and Lafferty, John},
	date = {2001},
	keywords = {basic concepts, {citeDiss}, {citeDissProp}}
}

@article{alonso_crowdsourcing_2008,
	title = {Crowdsourcing for relevance evaluation},
	volume = {42},
	issn = {0163-5840},
	doi = {10.1145/1480506.1480508},
	abstract = {Relevance evaluation is an essential part of the development and maintenance of information retrieval systems. Yet traditional evaluation approaches have several limitations; in particular, conducting new editorial evaluations of a search system can be very expensive. We describe a new approach to evaluation called {TERC}, based on the crowdsourcing paradigm, in which many online users, drawn from a large community, each performs a small evaluation task.},
	pages = {9--15},
	number = {2},
	journaltitle = {{SIGIR} Forum},
	author = {Alonso, Omar and Rose, Daniel E. and Stewart, Benjamin},
	date = {2008-11},
	keywords = {candidates, {citeDiss}, {citeDissProp}, crowdsourcing, humans in {IR}, Miles suggestions}
}

@article{mason_financial_2010,
	title = {Financial incentives and the "performance of crowds"},
	volume = {11},
	issn = {1931-0145},
	doi = {10.1145/1809400.1809422},
	abstract = {The relationship between financial incentives and performance, long of interest to social scientists, has gained new relevance with the advent of web-based "crowd-sourcing" models of production. Here we investigate the effect of compensation on performance in the context of two experiments, conducted on Amazon's Mechanical Turk ({AMT}). We find that increased financial incentives increase the quantity, but not the quality, of work performed by participants, where the difference appears to be due to an "anchoring" effect: workers who were paid more also perceived the value of their work to be greater, and thus were no more motivated than workers paid less. In contrast with compensation levels, we find the details of the compensation scheme do matter--specifically, a "quota" system results in better work for less pay than an equivalent "piece rate" system. Although counterintuitive, these findings are consistent with previous laboratory studies, and may have real-world analogs as well.},
	pages = {100--108},
	number = {2},
	journaltitle = {{SIGKDD} Explor. Newsl.},
	author = {Mason, Winter and Watts, Duncan J.},
	date = {2010-05},
	keywords = {candidates, {citeDiss}, {citeDissProp}, {citeiConf}14, crowd-sourcing, crowdsourcing, extrinsic motivation, hcir, humans in {IR}, incentives, Intrinsic motivation, mechanical turk, peer production, performance}
}

@report{urbano_university_2011,
	title = {The University Carlos {III} of Madrid  at {TREC} 2011 Crowdsourcing Track},
	author = {Urbano, Julián and Marrero, Mónica and Martín, Diego and Morato, Jorge and Robles, Karina and Lloréns, Juan},
	date = {2011},
	keywords = {{citeDiss}, {hcirCITE}}
}

@inproceedings{lease_overview_2011,
	title = {Overview of the {TREC} 2011 Crowdsourcing Track (Conference Notebook)},
	booktitle = {Text Retrieval Conference Notebook},
	author = {Lease, Matthew and Kazai, Gabriella},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {hcirCITE}, {TREC} crowd}
}

@inproceedings{hsueh_data_2009,
	location = {Stroudsburg, {PA}, {USA}},
	title = {Data quality from crowdsourcing: a study of annotation selection criteria},
	series = {{HLT} '09},
	shorttitle = {Data quality from crowdsourcing},
	abstract = {Annotation acquisition is an essential step in training supervised classifiers. However, manual annotation is often time-consuming and expensive. The possibility of recruiting annotators through Internet services (e.g., Amazon Mechanic Turk) is an appealing option that allows multiple labeling tasks to be outsourced in bulk, typically with low overall costs and fast completion rates. In this paper, we consider the difficult problem of classifying sentiment in political blog snippets. Annotation data from both expert annotators in a research lab and non-expert annotators recruited from the Internet are examined. Three selection criteria are identified to select high-quality annotations: noise level, sentiment ambiguity, and lexical uncertainty. Analysis confirm the utility of these criteria on improving data quality. We conduct an empirical study to examine the effect of noisy annotations on the performance of sentiment classification models, and evaluate the utility of annotation selection on classification accuracy and efficiency.},
	pages = {27--35},
	booktitle = {Proceedings of the {NAACL} {HLT} 2009 Workshop on Active Learning for Natural Language Processing},
	publisher = {Association for Computational Linguistics},
	author = {Hsueh, Pei-Yun and Melville, Prem and Sindhwani, Vikas},
	date = {2009},
	keywords = {{citeDiss}, {hcirCITE}}
}

@inproceedings{li_can_2009,
	title = {Can movies and books collaborate? cross-domain collaborative filtering for sparsity reduction},
	shorttitle = {Can movies and books collaborate?},
	pages = {2052--2057},
	booktitle = {Proceedings of the 21st international jont conference on Artifical intelligence},
	author = {Li, Bin and Yang, Qiang and Xue, Xiangyang},
	date = {2009},
	file = {[PDF] from ijcai.org:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\D9E2CP73\\Li et al. - 2009 - Can movies and books collaborate cross-domain col.pdf:application/pdf}
}

@inproceedings{donmez_probabilistic_2010,
	title = {A probabilistic framework to learn from multiple annotators with time-varying accuracy},
	abstract = {This paper addresses the challenging problem of learning from multiple annotators whose labeling accuracy (reliability) differs and varies over time. We propose a framework based on Sequential Bayesian Estimation to learn the expected accuracy at each time step while simultaneously deciding which annotators to query for a label in an incremental learning framework. We develop a variant of the particle filtering method that estimates the expected accuracy at every time step by sets of weighted samples and performs sequential Bayes updates. The estimated expected accuracies are then used to decide which annotators to be queried at the next time step. The empirical analysis shows that the proposed method is very effective at predicting the true label using only moderate labeling efforts, resulting in cleaner labels to train classifiers. The proposed method significantly outperforms a repeated labeling baseline which queries all labelers per example and takes the majority vote to predict the true label. Moreover, our method is able to track the true accuracy of an annotator quite well in the absence of gold standard labels. These results demonstrate the strength of the proposed method in terms of estimating the time-varying reliability of multiple annotators and producing cleaner, better quality labels without extensive label queries.},
	pages = {826--837},
	booktitle = {{SIAM} International Conference on Data Mining ({SDM})},
	author = {Donmez, Pinar and Carbonell, Jaime and Schneider, Jeff},
	date = {2010},
	keywords = {{citeDiss}, {hcirCITE}, {hcirMidtermCITE}},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\SZBMQBWD\\Donmez et al. - 2010 - A probabilistic framework to learn from multiple a.pdf:application/pdf}
}

@inproceedings{ipeirotis_quality_2010,
	location = {New York, {NY}, {USA}},
	title = {Quality management on Amazon Mechanical Turk},
	isbn = {978-1-4503-0222-7},
	doi = {10.1145/1837885.1837906},
	series = {{HCOMP} '10},
	pages = {64--67},
	booktitle = {Proceedings of the {ACM} {SIGKDD} Workshop on Human Computation},
	publisher = {{ACM}},
	author = {Ipeirotis, Panagiotis G. and Provost, Foster and Wang, Jing},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}}
}

@inproceedings{dekel_vox_2009,
	title = {Vox Populi: Collecting High-Quality Labels from a Crowd},
	abstract = {With the emergence of search engines and crowdsourcing
 websites, machine learning practitioners are faced with datasets that are labeled by a large heterogeneous set of teachers. These datasets test the limits of our existing learning theory, which largely assumes that data is sampled i.i.d. from a fixed distribution. In many cases, the number of teachers actually scales with the number of examples, with each teacher providing just a handful of labels, precluding any statistically reliable assessment of an individual teacher’s quality. In this paper, we study the problem of pruning low-quality teachers in a crowd, in order to improve the label quality of our training set. Despite the hurdles mentioned above, we show that this is in fact achievable with a simple and efficient algorithm, which does not require that each example be repeatedly labeled by multiple teachers. We provide a theoretical analysis of our algorithm and back our findings with empirical evidence.},
	eventtitle = {{COLT} 2009},
	author = {Dekel, Ofer and Shamir, Ohad},
	date = {2009-06-24},
	keywords = {{citeDiss}, {hcirCITE}}
}

@inproceedings{raykar_supervised_2009,
	location = {New York, {NY}, {USA}},
	title = {Supervised learning from multiple experts: whom to trust when everyone lies a bit},
	isbn = {978-1-60558-516-1},
	doi = {10.1145/1553374.1553488},
	series = {{ICML} '09},
	shorttitle = {Supervised learning from multiple experts},
	pages = {889--896},
	booktitle = {Proceedings of the 26th Annual International Conference on Machine Learning},
	publisher = {{ACM}},
	author = {Raykar, Vikas C. and Yu, Shipeng and Zhao, Linda H. and Jerebko, Anna and Florin, Charles and Valadez, Gerardo Hermosillo and Bogoni, Luca and Moy, Linda},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{eickhoff_increasing_2012,
	title = {Increasing cheat robustness of crowdsourcing tasks},
	issn = {1386-4564, 1573-7659},
	doi = {10.1007/s10791-011-9181-9},
	abstract = {Crowdsourcing successfully strives to become a widely used means of collecting large-scale scientific corpora. Many research fields, including Information Retrieval, rely on this novel way of data acquisition. However, it seems to be undermined by a significant share of workers that are primarily interested in producing quick generic answers rather than correct ones in order to optimise their time-efficiency and, in turn, earn more money. Recently, we have seen numerous sophisticated schemes of identifying such workers. Those, however, often require additional resources or introduce artificial limitations to the task. In this work, we take a different approach by investigating means of a priori making crowdsourced tasks more resistant against cheaters.},
	journaltitle = {Information Retrieval},
	author = {Eickhoff, Carsten and Vries, Arjen P.},
	date = {2012-02-14},
	keywords = {{citeDiss}, {citeDissProp}, {hcirCITE}, {hcirMidtermCITE}},
	file = {Eickhoff and Vries - 2012 - Increasing cheat robustness of crowdsourcing tasks.html:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PM8NP24R\\Eickhoff and Vries - 2012 - Increasing cheat robustness of crowdsourcing tasks.html:text/html}
}

@inproceedings{wallace_who_2011,
	title = {Who should label what? Instance allocation in multiple expert active learning},
	shorttitle = {Who should label what?},
	booktitle = {Proceedings of the {SIAM} International Conference on Data Mining ({SDM})},
	author = {Wallace, B. and Small, K. and Brodley, C. and Trikalinos, T.},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {hcirCITE}, {hcirMidtermCITE}},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\R4E4ACD6\\Wallace et al. - 2011 - Who should label what Instance allocation in mult.pdf:application/pdf}
}

@inproceedings{sheng_get_2008,
	location = {New York, {NY}, {USA}},
	title = {Get another label? improving data quality and data mining using multiple, noisy labelers},
	isbn = {978-1-60558-193-4},
	doi = {10.1145/1401890.1401965},
	series = {{KDD} '08},
	shorttitle = {Get another label?},
	pages = {614--622},
	booktitle = {Proceedings of the 14th {ACM} {SIGKDD} international conference on Knowledge discovery and data mining},
	publisher = {{ACM}},
	author = {Sheng, Victor S. and Provost, Foster and Ipeirotis, Panagiotis G.},
	date = {2008},
	keywords = {{citeDiss}, {citeDissProp}, data preprocessing, data selection, {hcirCITE}}
}

@inproceedings{welinder_multidimensional_2010,
	title = {The multidimensional wisdom of crowds},
	volume = {6},
	pages = {8},
	booktitle = {Neural Information Processing Systems Conference ({NIPS})},
	author = {Welinder, P. and Branson, S. and Belongie, S. and Perona, P.},
	date = {2010},
	keywords = {{citeDiss}},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\E6SSEEP6\\Welinder et al. - 2010 - The multidimensional wisdom of crowds.pdf:application/pdf}
}

@inproceedings{welinder_online_2010,
	title = {Online crowdsourcing: Rating annotators and obtaining cost-effective labels},
	isbn = {978-1-4244-7029-7},
	doi = {10.1109/CVPRW.2010.5543189},
	shorttitle = {Online crowdsourcing},
	abstract = {Labeling large datasets has become faster, cheaper, and easier with the advent of crowdsourcing services like Amazon Mechanical Turk. How can one trust the labels obtained from such services? We propose a model of the labeling process which includes label uncertainty, as well a multi-dimensional measure of the annotators' ability. From the model we derive an online algorithm that estimates the most likely value of the labels and the annotator abilities. It finds and prioritizes experts when requesting labels, and actively excludes unreliable annotators. Based on labels already obtained, it dynamically chooses which images will be labeled next, and how many labels to request in order to achieve a desired level of confidence. Our algorithm is general and can handle binary, multi-valued, and continuous annotations (e.g. bounding boxes). Experiments on a dataset containing more than 50,000 labels show that our algorithm reduces the number of labels required, and thus the total cost of labeling, by a large factor while keeping error rates low on a variety of datasets.},
	eventtitle = {2010 {IEEE} Computer Society Conference on Computer Vision and Pattern Recognition Workshops ({CVPRW})},
	pages = {25--32},
	booktitle = {2010 {IEEE} Computer Society Conference on Computer Vision and Pattern Recognition Workshops ({CVPRW})},
	publisher = {{IEEE}},
	author = {Welinder, P. and Perona, P.},
	date = {2010-06-13},
	keywords = {Adaptation model, amazon mechanical turk, {citeDiss}, {citeDissProp}, Computer vision, cost effective label, costing, Costs, dataset labeling service, Error analysis, {hcirCITE}, {hcirMidtermCITE}, image classification, Labeling, label uncertainty, multidimensional annotator ability measurement, multivalued continuous annotation, Noise figure, online crowdsourcing, Outsourcing, rating annotator, Web services}
}

@inproceedings{grady_crowdsourcing_2010,
	location = {Stroudsburg, {PA}, {USA}},
	title = {Crowdsourcing document relevance assessment with Mechanical Turk},
	series = {{CSLDAMT} '10},
	pages = {172--179},
	booktitle = {Proceedings of the {NAACL} {HLT} 2010 Workshop on Creating Speech and Language Data with Amazon's Mechanical Turk},
	publisher = {Association for Computational Linguistics},
	author = {Grady, Catherine and Lease, Matthew},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{whitehill_whose_2009,
	title = {Whose vote should count more: Optimal integration of labels from labelers of unknown expertise},
	volume = {22},
	shorttitle = {Whose vote should count more},
	pages = {2035--2043},
	journaltitle = {Advances in Neural Information Processing Systems},
	author = {Whitehill, J. and Ruvolo, P. and Wu, T. and Bergsma, J. and Movellan, J.},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, {hcirCITE}, {hcirMidtermCITE}},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PV9EWG5K\\Whitehill et al. - 2009 - Whose vote should count more Optimal integration .pdf:application/pdf}
}

@article{moyle_manuscript_2010,
	title = {Manuscript transcription by crowdsourcing: Transcribe Bentham},
	volume = {20},
	shorttitle = {Manuscript transcription by crowdsourcing},
	abstract = {Transcribe Bentham is testing the feasibility of outsourcing the work of manuscript transcription to members of the public.  {UCL} Library Services holds 60,000 folios of manuscripts of the philosopher and jurist Jeremy Bentham (1748-1832).  Transcribe Bentham will digitise 12,500 Bentham folios, and, through a wiki-based interface, allow volunteer transcribers to take temporary ownership of manuscript images and to create {TEI}-encoded transcription text for final approval by {UCL} experts.  Approved transcripts will be stored and preserved, with the manuscript images, in {UCL}'s public Digital Collections repository.  

The project makes innovative use of traditional Library material. It will stimulate public engagement with {UCL}'s scholarly archive collections and the challenges of palaeography and manuscript transcription; it will raise the profile of the work and thought of Jeremy Bentham; and it will create new digital resources for future use by  professional researchers.  Towards the end of the project, the transcription tool will be made available to other projects and services.},
	number = {3},
	journaltitle = {{LIBER} Quarterly},
	author = {Moyle, M. and Tonra, J. and Wallace, V.},
	date = {2010},
	keywords = {{citeDiss}, {citeDissProp}, crowdsourcing, digital humanities, {swiftCite}},
	file = {Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\TDIV9BMK\\20474.html:text/html}
}

@article{golder_structure_2007,
	title = {The Structure of Collaborative Tagging Systems},
	author = {Golder and Huberman},
	date = {2007},
	keywords = {{citeDiss}},
	file = {Golder_Huberman_2007_The Structure of Collaborative Tagging Systems.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\2007\\Golder_Huberman_2007_The Structure of Collaborative Tagging Systems.pdf:application/pdf}
}

@report{holley_many_2009,
	title = {Many Hands Make Light Work: Public Collaborative {OCR} Text Correction in Australian Historic Newspapers},
	abstract = {The {ANDP} team had from the outset in January 2007 decided to make a considerable investment in software development in order to be able to quality assure digital outputs to ensure they met minimum standards and to future proof the files in case they could be improved further in the future.

Contributors had all expressed concern that the digital outputs (image quality, {OCR} text) may not be good enough to enable adequate full text retrieval or to meet user expectations. The {ANDP} team had regularly brainstormed and reviewed ideas to improve the quality of outputs and had implemented a number of ideas to achieve this3. At this stage the team were assuming that quality of data entirely relied on the Library’s digitisation technique. It had not yet occurred to the team or the contributors that the public may play a role in improving and enhancing the quality of the data.},
	institution = {National Library of Australia},
	author = {Holley, Rose},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, {ICcited}, sorted}
}

@inproceedings{spiteri_social_2011,
	title = {Social discovery tools: Cataloguing meets user convenience},
	volume = {3},
	abstract = {This paper examines how library users access, use, and interact with two social discovery systems used in two Canadian public library systems. How do public library users interact with social discovery systems? How does usage between the two social discovery systems compare? Daily transaction logs of the social discovery systems used by the two libraries were compiled from May-August, 2010. Fifty sets of bibliographic records were compared to evaluate user-contributed content. Results indicate that features that allow for user-generated content are underused in both systems. Future research will thus focus on clients' motivations for engaging with the social features of social discovery systems, and their perceptions of, and satisfaction with, the benefits of these features.},
	booktitle = {Proceedings from North American Symposium on Knowledge Organization},
	author = {Spiteri, Louise F.},
	date = {2011},
	keywords = {bibliocommons, {citeDiss}, {citeDissProp}, {ICcited}, libraries, opac, sorted}
}

@online{_lists_,
	title = {Lists},
	titleaddon = {Bibliocommons},
	urldate = {2011-10-11},
	keywords = {{citeDiss}, {citeDissProp}, {ICcited}, sorted}
}

@inproceedings{sen_tagging_2006,
	location = {New York, {NY}, {USA}},
	title = {tagging, communities, vocabulary, evolution},
	isbn = {1-59593-249-6},
	doi = {10.1145/1180875.1180904},
	series = {{CSCW} '06},
	pages = {181--190},
	booktitle = {Proceedings of the 2006 20th anniversary conference on Computer supported cooperative work},
	publisher = {{ACM}},
	author = {Sen, Shilad and Lam, Shyong K. and Rashid, Al Mamunur and Cosley, Dan and Frankowski, Dan and Osterhouse, Jeremy and Harper, F. Maxwell and Riedl, John},
	date = {2006},
	keywords = {{citeDiss}, Communities, evolution, social book-marking, tagging, vocabulary},
	file = {Sen et al_2006_Tagging, Communities, Vocabulary, Evolution.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM2006\\Sen et al_2006_Tagging, Communities, Vocabulary, Evolution.pdf:application/pdf}
}

@inproceedings{quinn_human_2011,
	title = {Human computation},
	isbn = {978-1-4503-0228-9},
	doi = {10.1145/1978942.1979148},
	pages = {1403},
	publisher = {{ACM} Press},
	author = {Quinn, Alexander J. and Bederson, Benjamin B.},
	date = {2011},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, crowdsourcing, data mining, human computation, literature review, social computing, survey, Taxonomy},
	file = {Quinn_Bederson_2011_Human computation.pdf:C\:\\Users\\Peter\\Dropbox\\papers\\ACM Press2011\\Quinn_Bederson_2011_Human computation.pdf:application/pdf;Shibboleth Authentication Request:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\K8IRQATF\\login.html:text/html}
}

@thesis{organisciak_why_2010,
	location = {Edmonton, Alberta},
	title = {Why Bother? Examining the motivations of users in large-scale crowd-powered online initiatives},
	url = {http://hdl.handle.net/10048/1370},
	abstract = {This study examines the motivations of participants in networked, large-scale content production and research – a paradigm of distributed work magnified by the Internet. This has come to be called crowdsourcing. The approach taken in examining the crowdsourcing paradigm is of retrospection, with a study focused on observed examples and existing theories. Thirteen cases of existing crowdsourcing sites were selected for study, from a larger sample of 300. These cases were coded by their site properties and analyzed, identifying possible motivational mechanisms. Subsequent interviews with eight medium to heavy Internet users further explored these features, with an emphasis on ranking relative importance of various motivators. This study concludes with a series of recommendations on motivating crowds in such projects, emphasizing among others the importance of topical interest, ease of participation, and appeals to the individuals’ knowledge. In addition to base motivators, a number of support, or secondary, motivators are outlined.},
	pagetotal = {167},
	institution = {University of Alberta},
	type = {Thesis},
	author = {Organisciak, Peter},
	date = {2010-08-31},
	keywords = {{citeDiss}, {citeDissProp}, {citeiConf}14, crowdsourcing, {ICcited}, motivation, sorted}
}

@book{shirky_here_2009,
	title = {Here comes everybody},
	publisher = {Penguin Books},
	author = {Shirky, C.},
	date = {2009},
	keywords = {{citeDiss}, {citeDissProp}, cognitive surplus, gamification, {ICcited}, sorted},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\BP8E9DKC\\Shirky - 2009 - Here comes everybody.pdf:application/pdf}
}

@inproceedings{snow_cheap_2008,
	location = {Stroudsburg, {PA}, {USA}},
	title = {Cheap and fast—but is it good?: evaluating non-expert annotations for natural language tasks},
	series = {{EMNLP} '08},
	shorttitle = {Cheap and fast—but is it good?},
	abstract = {Human linguistic annotation is crucial for many natural language processing tasks but can be expensive and time-consuming. We explore the use of Amazon's Mechanical Turk system, a significantly cheaper and faster method for collecting annotations from a broad base of paid non-expert contributors over the Web. We investigate five tasks: affect recognition, word similarity, recognizing textual entailment, event temporal ordering, and word sense disambiguation. For all five, we show high agreement between Mechanical Turk non-expert annotations and existing gold standard labels provided by expert labelers. For the task of affect recognition, we also show that using non-expert labels for training machine learning algorithms can be as effective as using gold standard annotations from experts. We propose a technique for bias correction that significantly improves annotation quality on two tasks. We conclude that many large labeling tasks can be effectively designed and carried out in this method at a fraction of the usual expense.},
	pages = {254--263},
	booktitle = {Proceedings of the Conference on Empirical Methods in Natural Language Processing},
	publisher = {Association for Computational Linguistics},
	author = {Snow, R. and O'Connor, B. and Jurafsky, D. and Ng, A.Y.},
	date = {2008},
	keywords = {annotation, {citeDiss}, {citeDissProp}, expertise, experts, {hcirCITE}, {hcirMidtermCITE}, mechanical turk, {mturkCITE}, tagging, turk},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\BQTJ4D2F\\login.html:text/html}
}

@inproceedings{novotney_cheap_2010,
	location = {Stroudsburg, {PA}, {USA}},
	title = {Cheap, fast and good enough: Automatic speech recognition with non-expert transcription},
	shorttitle = {Cheap, fast and good enough},
	eventtitle = {{HLT} '10},
	pages = {207--215},
	booktitle = {Human Language Technologies: The 2010 Annual Conference of the North American Chapter of the Association for Computational Linguistics},
	author = {Novotney, S. and Callison-Burch, C.},
	date = {2010},
	keywords = {annotation, {citeDiss}, {citeDissProp}, expertise, experts, {hcirCITE}, {hcirMidtermCITE}, mechanical turk, {mturkCITE}, turk},
	file = {Google Scholar Linked Page:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\4P7694FR\\login.html:text/html}
}

@inproceedings{bernstein_soylent_2010,
	location = {New York, {NY}},
	title = {Soylent: a word processor with a crowd inside.},
	isbn = {978-1-4503-0271-5},
	doi = {10.1145/1866029.1866078},
	eventtitle = {{UIST} '10},
	pages = {313--322},
	booktitle = {Proceedings of the 23nd annual {ACM} symposium on User interface software and technology},
	publisher = {{ACM} Press},
	author = {Bernstein, Michael S. and Little, Greg and Miller, Robert C. and Hartmann, Björn and Ackerman, Mark S. and Karger, David R. and Crowell, David and Panovich, Katrina},
	date = {2010},
	keywords = {candidates, {citeDiss}, {citeDissProp}, favorites, hcir, {mturkCITE}, rate4, {swiftCite}},
	file = {Soylent:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\6T3DT5QQ\\citation.html:text/html}
}

@article{von_ahn_games_2006,
	title = {Games with a purpose},
	volume = {39},
	issn = {0018-9162},
	abstract = {Through online games, people can collectively solve large-scale computational problems.},
	pages = {96--98},
	number = {6},
	journaltitle = {Computer},
	author = {von Ahn, Luis},
	date = {2006-06},
	keywords = {{citeDiss}, {citeDissProp}, gamification, serious games},
	file = {ieee-gwap.pdf:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\9UGX3936\\ieee-gwap.pdf:application/pdf}
}

@online{_what_,
	title = {What is {reCAPTCHA}?},
	url = {http://recaptcha.net/learnmore.html},
	titleaddon = {Recaptcha},
	urldate = {2008-09-27},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {What is reCAPTCHA?:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\M3CIH96B\\learnmore.html:text/html}
}

@inproceedings{ahn_labeling_2004,
	location = {Vienna, Austria},
	title = {Labeling images with a computer game},
	isbn = {1-58113-702-8},
	abstract = {We introduce a new interactive system: a game that is fun and can be used to create valuable output. When people play the game they help determine the contents of images by providing meaningful labels for them. If the game is played as much as popular online games, we estimate that most images on the Web can be labeled in a few months. Having proper labels associated with each image on the Web would allow for more accurate image search, improve the accessibility of sites (by providing descriptions of images to visually impaired individuals), and help users block inappropriate images. Our system makes a significant contribution because of its valuable output and because of the way it addresses the image-labeling problem. Rather than using computer vision techniques, which don't work well enough, we encourage people to do the work by taking advantage of their desire to be entertained.},
	pages = {319--326},
	booktitle = {Proceedings of the {SIGCHI} conference on Human factors in computing systems},
	publisher = {{ACM}},
	author = {von Ahn, Luis and Dabbish, Laura},
	date = {2004},
	keywords = {{citeDiss}, {citeDissProp}, distributed knowledge acquisition, hcir, humans in {IR}, {ICcited}, image labeling, online games, rate5, World Wide Web}
}

@book{howe_crowdsourcing_2008,
	edition = {1},
	title = {Crowdsourcing: Why the Power of the Crowd Is Driving the Future of Business},
	isbn = {0-307-39620-7},
	shorttitle = {Crowdsourcing},
	pagetotal = {320},
	publisher = {Crown Business},
	author = {Howe, Jeff},
	date = {2008-08-26},
	keywords = {{citeDiss}, {citeDissProp}}
}

@book{mackay_memoirs_1852,
	title = {Memoirs of Extraordinary Popular Delusions and the Madness of Crowds},
	pagetotal = {503},
	author = {Mackay, Charles},
	date = {1852},
	keywords = {{citeDiss}, {citeDissProp}}
}

@book{surowiecki_wisdom_2004,
	title = {The Wisdom of Crowds},
	publisher = {Doubleday},
	author = {Surowiecki, James},
	date = {2004},
	keywords = {{citeDiss}, {citeDissProp}, crowdsourcing, {ICcited}, read, sorted}
}

@book{benkler_wealth_2006,
	location = {New Haven},
	title = {Wealth of Networks},
	publisher = {Yale University Press},
	author = {Benkler, Yochai},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}},
	file = {Benkler_Wealth_Of_Networks.pdf:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\EATAADGC\\Benkler_Wealth_Of_Networks.pdf:application/pdf}
}

@online{howe_crowdsourcing_2006,
	title = {Crowdsourcing: A Definition},
	url = {http://crowdsourcing.typepad.com/cs/2006/06/crowdsourcing_a.html},
	shorttitle = {Crowdsourcing},
	titleaddon = {Crowdsourcing: Tracking the rise of the amateur},
	author = {Howe, Jeff},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{howe_rise_2006,
	title = {The rise of crowdsourcing},
	volume = {14},
	number = {6},
	journaltitle = {Wired Magazine},
	author = {Howe, Jeff},
	date = {2006},
	keywords = {{citeDiss}, {citeDissProp}}
}

@book{le_bon_crowd_1896,
	title = {The Crowd: A Study of the Popular Mind},
	author = {Le Bon, Gustav},
	date = {1896},
	keywords = {{citeDiss}, {citeDissProp}}
}

@article{lakhani_how_2003,
	title = {How open source software works: "free" user-to-user assistance},
	volume = {32},
	doi = {10.1016/S0048-7333(02)00095-1},
	shorttitle = {How open source software works},
	abstract = {Research into free and open source software development projects has so far largely focused on how the major tasks of software development are organized and motivated. But a complete project requires the execution of "mundane but necessary" tasks as well. In this paper, we explore how the mundane but necessary task of field support is organized in the case of Apache web server software, and why some project participants are motivated to provide this service gratis to others. We find that the Apache field support system functions effectively. We also find that, when we partition the help system into its component tasks, 98\% of the effort expended by information providers in fact returns direct learning benefits to those providers. This finding considerably reduces the puzzle of why information providers are willing to perform this task "for free." Implications are discussed.},
	pages = {923--943},
	number = {6},
	journaltitle = {Research Policy},
	author = {Lakhani, Karim R. and Hippel, Eric von},
	date = {2003-06},
	keywords = {{citeDiss}, {citeDissProp}, Open source software, User innovation, User support, Virtual community},
	file = {ScienceDirect Snapshot:C\:\\Users\\Peter\\AppData\\Roaming\\Zotero\\Zotero\\Profiles\\zec9madg.default\\zotero\\storage\\PPCWABCT\\science.html:text/html}
}